{"acc_norm":0.31313993174061433,"acc_norm_definition":"argmax summed conditional likelihood / answer character length, excluding leading delimiter","batch_policy":{"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.2613610369153321},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.048841834999620914},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.05111184902489185},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.05204826593399048},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.05268274899572134},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.0527509959647432},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.053355883923359215},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.052864017081446946},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05303652898874134},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.0537622319534421},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05338542896788567},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.053172782994806767},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.05724984605330974},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.05843410501256585},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.058204786968417466},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.05913656495977193},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06338042905554175},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.060753540019504726},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06040413794107735},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.05852557497564703},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.05828305194154382},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.05862093297764659},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.059149730019271374},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.05959766497835517},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.0590052199549973},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.05933415307663381},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.05933539499528706},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06133415293879807},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.05934850999619812},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06107554095797241},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.059622099972330034},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.061330034979619086},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06078788498416543},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06044039595872164},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06219528894871473},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06006637797690928},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06561190111096948},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.0666036510374397},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.0651920479722321},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06461404997389764},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06495872291270643},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06545139104127884},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06463760405313224},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06472811195999384},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06440234498586506},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06764336000196636},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06736402900423855},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06802181503735483},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.0679930109763518},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.0674678219947964},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06623880995903164},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06533476407639682},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06575140100903809},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.0676946040475741},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06937736296094954},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06814261095132679},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06793206790462136},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06779228697996587},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06830330996308476},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06843356299214065},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06905486097093672},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06672141002491117},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06815979303792119},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06919338996522129},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0681302270386368},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0679794360185042},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06694205105304718},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06995661300607026},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06706611206755042},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0682542179711163},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06860609399154782},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06833920709323138},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06804424710571766},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06868552207015455},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06891431892290711},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06861272698733956},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06934188702143729},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06860918994061649},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07166618201881647},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06841074989642948},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07160879403818399},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07232779602054507},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07168567692860961},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07389384601265192},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07241094997152686},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07168128900229931},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07305626396555454},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07228885206859559},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07201235694810748},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07218319503590465},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.0728652480756864},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.0737669289810583},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07357833697460592},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07444796897470951},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07303889095783234},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07542102399747819},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07427436497528106},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07407624600455165},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07301994599401951},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.0767905079992488},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07328865607269108},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07751401700079441},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07610237505286932},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07517371303401887},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07522276393137872},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07338108005933464},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07509815192315727},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07414306094869971},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07508898398373276},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.0735889570787549},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07550364802591503},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07594898797105998},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07579151389654726},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07467508199624717},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07820640900172293},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07727484102360904},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07781501905992627},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.0780470910249278},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07772712397854775},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07924884301610291},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07797324296552688},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07940107595641166},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07819771196227521},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07743343699257821},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.08037136599887162},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.08123243099544197},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.0791988589335233},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07771515205968171},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08341576298698783},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.0812851000810042},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08048537792637944},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08043081709183753},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.07893793191760778},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.07863629108760506},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08100011805072427},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.0812504249624908},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08091644197702408},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07932121690828353},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08082907903008163},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07925180799793452},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07883224589750171},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08131839195266366},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08668020099867135},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.09034903300926089},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08459302899427712},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08833749208133668},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08920096198562533},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.09057399199809879},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08919064304791391},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08650007494725287},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08773370599374175},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08672711905092001},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08876519999466836},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08959929505363107},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.09012750000692904},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.08696178602986038},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.09111538203433156},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.08889667503535748},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.08955091505777091},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.0883225139696151},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.09131316200364381},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.09012272104155272},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.09416173293720931},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.09021005500108004},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.089950290042907},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.0867856879485771},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.08897557202726603},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.0892340641003102},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.0956972970161587},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.08728078193962574},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.09221596491988748},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09257712808903307},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09449945401865989},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.0950278090313077},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09103225800208747},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09205427300184965},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09715669904835522},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09405882190912962},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.08898559107910842},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.08828152797650546},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.08946272102184594},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09090954798739403},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09724613500293344},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09655860404018313},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09530509600881487},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.0933862280799076},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.09353099798317999},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.09042951290030032},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.08906184195075184},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09767344198189676},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.1006317000137642},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.0990669809980318},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09706456598360091},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09844084409996867},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.099419339094311},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.10178856702987105},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.10191327007487416},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.09860437898896635},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.10253974702209234},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.10463002696633339},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.10101408301852643},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.10050444200169295},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.0962112860288471},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.09770258702337742},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.09737441490869969},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.09906156198121607},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.10018451500218362},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.10158784303348511},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.10187253402546048},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.09972533001564443},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.09827715798746794},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.10629562300164253},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1077975130174309},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.11102895892690867},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.10685820202343166},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.11034915200434625},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.11073278693947941},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.11034063599072397},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10898985690437257},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10968870006036013},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.1054963730275631},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.11056218901649117},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.11125355900730938},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.11193263495806605},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.11136183002963662},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.11116103606764227},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.11401567992288619},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.11598537291865796},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.11366662802174687},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11530544504057616},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1132480880478397},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11343980499077588},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11617917497642338},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11866345698945224},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11698577890638262},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11927582998760045},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.1195608910638839},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11814040201716125},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11403344199061394},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11501651594880968},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.1166226320201531},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11901396093890071},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11694987094961107},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11625877197366208},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.1144199640257284},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11560991301666945},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.11491127999033779},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.11730595608241856},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.12053707207087427},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.12476803199388087},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.12033338204491884},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.11850604403298348},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.12160017399583012},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.12576798698864877},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.12075714208185673},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.13153869798406959},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.12708176707383245},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.12794658006168902},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.12703000998590142},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.13509942998643965},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.12797190702985972},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.13071981095708907},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.1262935670092702},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.1351761519908905},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.13587329199071974},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.13782423990778625},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.1372038940899074},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.13629279402084649},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.13507896196097136},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.1557756579713896},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.15770252200309187},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.15733073907904327},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.15274877392221242},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.15815948403906077},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.16055031295400113},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.17039354995358735},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.1720279579749331},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.17440332705155015},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.17551253596320748},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.1765509129036218},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.17124606005381793},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.17968696099705994},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.17482114699669182},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.18053718702867627},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.17794932005926967},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.19076310691889375},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.20516099804081023},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.19557025202084333},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.20158980798441917},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.2233856349484995},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.2218340819235891},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.2518089619698003},{"candidates":15,"kind":"choice","padded_tokens":2685,"seconds":0.3358509639510885}],"chat_template":false,"data_loading_seconds":0.0681985350092873,"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa"},"device":"cuda:0","embedding_head_tied":true,"evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"qwen35-arc-challenge-v1","finished":1792212395,"gpu":1,"id":"a21f95e1e94c8466257d4fd3","images_supplied":false,"loaded_class":"Qwen3_5ForConditionalGeneration","loader_ignored_auxiliary_patterns":["^mtp.*"],"loading_info":{"error_msgs":[],"mismatched_keys":[],"missing_keys":[],"unexpected_keys":[]},"measured_cost_usd":null,"metric":"accuracy","model":"SeerRay-Lab/Xiaomi-OCR-0","model_load_seconds":3.236276131006889,"model_revision":"ff990be318c4c68f7e621dd690231200f47e48c4","n":1172,"notes":"Full raw-accuracy text-only diagnostic for a same-base Qwen3.5 cohort. Stock trusted model classes, no chat or images; generic text scores do not validate specialist OCR/classification/retrieval uses. Unique total/text/vision parameters measured after loading; source-reported metrics are not imported.","optional_kernel_packages":{"causal_conv1d":false,"fla":false,"flash_attn":false,"kernels":false},"parameters":852985920,"peak_gpu_bytes":4513547264,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Challenge / Qwen3.5 same-base cohort","catalogue_id":"arc-challenge","choice_score":"sum_log_likelihood","contract":"Question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"qwen35-arc-challenge-full","license":["cc-by-sa-4.0"],"limit":1172,"metric":"accuracy","models":[{"id":"Qwen/Qwen3.5-0.8B-Base","revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68"},{"id":"Qwen/Qwen3.5-0.8B","revision":"2fc06364715b967f1860aea9cf38778875588b17"},{"id":"Mapika/decider-0.8b","revision":"a0a01d6f8135298f400a8c856b355793012ae971"},{"id":"SeerRay-Lab/Xiaomi-OCR-0","revision":"ff990be318c4c68f7e621dd690231200f47e48c4"},{"id":"unsloth/Qwen3.5-0.8B-Base","revision":"44bc982576bf30f0a9440303598859d92604ea5d"},{"id":"nineninesix/qwen3_5-full-attn-only-14","revision":"a26dff91d90e168da8e49f0c19dde98447c095da"},{"id":"OmniJev/PlayJev-0.8B","revision":"a7348002b1e159add7037d6d50812cd4db2d96e9"},{"id":"mithulaartigala/reasoning-summarizer-qwen3.5-0.8b-preview","revision":"558fc651bf01bbe7cf306ebf2fb32032d7b4e818"},{"id":"sh0ck0r/Qwen3.5-0.8B-heretic","revision":"fb35b60374fc15c99a3890e55a5a77e6d015cf4e"},{"id":"McGill-NLP/AfriqueQwen3.5-0.8B-50Langs","revision":"f49099d79e51bbb783fae622877c4c3415142c82"},{"id":"ahmedeltawel/Qwen3.5-0.8B-hsb-dsb","revision":"7e781a3e970474d3fcb215ec0d3c5832e1131b9f"},{"id":"Kewk/Heretical-Qwen3.5-0.8B","revision":"c16807de35d1a569fecbdb240eaaa0ea332372fb"},{"id":"cooperdk/BooruNL-0.8B","revision":"14dbcf2e009f77697dcc7f23e2a706b71b52a5d1"},{"id":"mghafiri/qwen3.5-0.8B-decision-model","revision":"4a9939034a27006b4b62ad5aed65a90b82d0d350"},{"id":"NamanAgnih0tri/AlphaRoute-0.8B-v1.5","revision":"ef87d71f1105f47a15d0d290ee000bcfc0ed68c0"}],"name":"Qwen3.5 0.8B cohort / full ARC-Challenge","parent_task":"sub1b-arc-challenge","priority_models":["Qwen/Qwen3.5-0.8B-Base"],"protocol_version":"codesota-qwen35-arc-challenge-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Challenge/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_qwen35_challenge.py","selection":"All 1172 test items; raw summed conditional answer likelihood; no chat or images; same frozen 14 candidates plus original base control","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":33.31652617303189,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","scope":"8877f1f73ee0a3c138937e07b7f6ca917f2c1d914d223e6a8ffa42b9ac8f0a47","score":0.2713310580204778,"scoring_seconds":28.152255935943685,"seconds":31.65009331703186,"seed":42,"selection":"Full pinned ARC-Challenge test split; all 1172 source-order items","status":"completed","task":"qwen35-arc-challenge-full","text_parameters":752393024,"tf32":false,"truncated_candidates":0,"vision_parameters":100592896,"worker":"white-gpu1"}