{"acc_norm":0.3660409556313993,"acc_norm_definition":"argmax summed conditional likelihood / answer character length, excluding leading delimiter","batch_policy":{"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.27743248792830855},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.051451391889713705},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.06169203098397702},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.055672654998488724},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.057629003073088825},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.05636988498736173},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.057450701016932726},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.06338286306709051},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05814990494400263},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.06266606692224741},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05781507107894868},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.06266966403927654},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.061623151996172965},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.07070324604865164},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.07834258198272437},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.062115299981087446},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06841135094873607},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.062081596930511296},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.07145506807137281},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.062052894034422934},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06824091402813792},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.0623267040355131},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06829238997306675},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06343444006051868},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06857596896588802},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06379008304793388},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06865412404295057},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.07270116196013987},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06899950793012977},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.07224185508675873},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06369731994345784},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.07130827405489981},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06400155695155263},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.07033663196489215},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06551847606897354},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06917625805363059},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07610175292938948},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07087208097800612},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.0792552640195936},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06778205803129822},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07440628099720925},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07137431704904884},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.0771387069253251},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07571651495527476},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06898728501982987},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07946064707357436},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07963559404015541},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07084708393085748},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07943774398881942},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07077148300595582},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07583653798792511},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07675389002542943},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.0688245720230043},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07837630494032055},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07463120005559176},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.08388562907930464},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07902803109027445},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07309069798793644},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07938639901112765},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0735739489318803},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07763552397955209},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07706980791408569},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07154412404634058},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0823944880394265},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07862089201807976},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07172730495221913},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07704501191619784},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07525949191767722},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07846086204517633},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.08007202798034996},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07357903895899653},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.0825313130626455},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07853917905595154},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07434324303176254},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07920767494942993},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07332978409249336},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.08129381597973406},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07937112997751683},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07434063800610602},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07923802197910845},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08391110692173243},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07829165703151375},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.0844017619965598},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08729789301287383},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08109245100058615},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08281179703772068},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.0854290269780904},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07724717899691314},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08323002804536372},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08228294108994305},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07701303297653794},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08550684305373579},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.0866167830536142},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08167576906271279},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08494334190618247},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08679962391033769},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.0820549859199673},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08538918395061046},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08695247897412628},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08052417100407183},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08764082298148423},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08849445497617126},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08235637692268938},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.0902362329652533},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08296329900622368},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07824041205458343},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.09055492701008916},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08877226302865893},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07941760693211108},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08522253204137087},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.09122152102645487},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08870197203941643},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07967860496137291},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08780490898061544},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.09273321903310716},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.08254654298070818},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.08854140201583505},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.09036025393288583},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.08401454996783286},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.092035848996602},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.09162973193451762},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.09248428593855351},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.08470539806876332},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.09138605697080493},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.09247002808842808},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.09302529401611537},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.08523814205545932},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.09220228891354054},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09376352094113827},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08892155101057142},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.0946419290266931},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09206380101386458},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.0946751810843125},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08477975707501173},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09414325002580881},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.09503887000028044},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.09176326100714505},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08381226297933608},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.09340897004585713},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.09481921000406146},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.09328492800705135},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08704708493314683},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10349054099060595},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1034673280082643},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10253377689514309},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10337686899583787},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.09601321700029075},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10422466008458287},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10542143299244344},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.10255245096050203},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.10454579896759242},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.10178812593221664},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.09701091691385955},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.1053064480656758},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.10680660500656813},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.1050474449293688},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.10468417697120458},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.10513605002779514},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.09658991207834333},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10521934495773166},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10793043102603406},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10620506294071674},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10665721690747887},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10645589197520167},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.09728673193603754},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10273511207196862},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10153078590519726},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10336536797694862},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10952252999413759},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10331236908677965},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10027448402252048},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10816389706451446},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11102993099484593},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10916560492478311},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.1034398659830913},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10602296399883926},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11215630196966231},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10262310307007283},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10480175702832639},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10428363003302366},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10590660700108856},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10901825898326933},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.11357414699159563},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10811881290283054},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10099131998140365},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.11114774201996624},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.10669052903540432},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.107044599018991},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.10511781496461481},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11379273398779333},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11632255197037011},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11376162595115602},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.1111223140032962},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.10886557505000383},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11520122003275901},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11774507502559572},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11646092007867992},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.1133231979329139},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11815540993120521},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11804221803322434},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11877591698430479},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11902776709757745},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.1123783360235393},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11149385699536651},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.112201186013408},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11126879695802927},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.10932620393577963},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11869627889245749},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11783466197084635},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11741498997434974},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11326647200621665},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12278638605494052},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12496188294608146},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12681209400761873},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12164086999837309},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.1274185039801523},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12707244895864278},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12660500698257238},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12407591100782156},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12471701600588858},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12645608000457287},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.1274009320186451},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12836628197692335},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12790227704681456},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12867010908666998},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12564759200904518},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.12946830701548606},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.1325217520352453},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13805150403641164},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1342657239874825},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.12897097994573414},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.13129029504489154},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.13126984692644328},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13205697492230684},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13145328999962658},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13777036999817938},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13847124692983925},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.1361049939878285},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.1349748669890687},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13541349407751113},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13342962507158518},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.1405002810060978},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.1363580860197544},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13341870508156717},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13393993698991835},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13441056397277862},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13605854799970984},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.1371248960494995},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.136105515062809},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.14220058207865804},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.14128884195815772},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.14406229392625391},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.14274677995126694},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.14427646389231086},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.14051509799901396},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.15034678403753787},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.14815103902947158},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.15403353003785014},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.14228667202405632},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.15709375601727515},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.1506686240900308},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.14974823698867112},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.1494748080149293},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.1521602360298857},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.15397177496924996},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.1584880059817806},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.16742593492381275},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.15386966394726187},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.15542023594025522},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.18467684602364898},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.1784543429967016},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.1876411649864167},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.1757540360558778},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.18867953098379076},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.18034306401386857},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.20035389298573136},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.19571075402200222},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.20653360697906464},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.2091454790206626},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.1993944039568305},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.2016052670078352},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.20640213205479085},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.2078835649881512},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.21318218705710024},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.20226845308206975},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.22186252498067915},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.24161075393203646},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.2303526879986748},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.2351354279089719},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.25831163907423615},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.261729734018445},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.29078948800452054},{"candidates":15,"kind":"choice","padded_tokens":2685,"seconds":0.39606519893277436}],"chat_template":false,"data_loading_seconds":0.048843148979358375,"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa"},"device":"cuda:0","embedding_head_tied":true,"evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"qwen35-arc-challenge-v1","finished":1792212426,"gpu":0,"id":"d9933c5cbfcdb9fd6ec4015f","images_supplied":false,"loaded_class":"Qwen3_5ForConditionalGeneration","loader_ignored_auxiliary_patterns":["^mtp.*"],"loading_info":{"error_msgs":[],"mismatched_keys":[],"missing_keys":[],"unexpected_keys":[]},"measured_cost_usd":null,"metric":"accuracy","model":"McGill-NLP/AfriqueQwen3.5-0.8B-50Langs","model_load_seconds":3.425090474076569,"model_revision":"f49099d79e51bbb783fae622877c4c3415142c82","n":1172,"notes":"Full raw-accuracy text-only diagnostic for a same-base Qwen3.5 cohort. Stock trusted model classes, no chat or images; generic text scores do not validate specialist OCR/classification/retrieval uses. Unique total/text/vision parameters measured after loading; source-reported metrics are not imported.","optional_kernel_packages":{"causal_conv1d":false,"fla":false,"flash_attn":false,"kernels":false},"parameters":852985920,"peak_gpu_bytes":4513547264,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Challenge / Qwen3.5 same-base cohort","catalogue_id":"arc-challenge","choice_score":"sum_log_likelihood","contract":"Question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"qwen35-arc-challenge-full","license":["cc-by-sa-4.0"],"limit":1172,"metric":"accuracy","models":[{"id":"Qwen/Qwen3.5-0.8B-Base","revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68"},{"id":"Qwen/Qwen3.5-0.8B","revision":"2fc06364715b967f1860aea9cf38778875588b17"},{"id":"Mapika/decider-0.8b","revision":"a0a01d6f8135298f400a8c856b355793012ae971"},{"id":"SeerRay-Lab/Xiaomi-OCR-0","revision":"ff990be318c4c68f7e621dd690231200f47e48c4"},{"id":"unsloth/Qwen3.5-0.8B-Base","revision":"44bc982576bf30f0a9440303598859d92604ea5d"},{"id":"nineninesix/qwen3_5-full-attn-only-14","revision":"a26dff91d90e168da8e49f0c19dde98447c095da"},{"id":"OmniJev/PlayJev-0.8B","revision":"a7348002b1e159add7037d6d50812cd4db2d96e9"},{"id":"mithulaartigala/reasoning-summarizer-qwen3.5-0.8b-preview","revision":"558fc651bf01bbe7cf306ebf2fb32032d7b4e818"},{"id":"sh0ck0r/Qwen3.5-0.8B-heretic","revision":"fb35b60374fc15c99a3890e55a5a77e6d015cf4e"},{"id":"McGill-NLP/AfriqueQwen3.5-0.8B-50Langs","revision":"f49099d79e51bbb783fae622877c4c3415142c82"},{"id":"ahmedeltawel/Qwen3.5-0.8B-hsb-dsb","revision":"7e781a3e970474d3fcb215ec0d3c5832e1131b9f"},{"id":"Kewk/Heretical-Qwen3.5-0.8B","revision":"c16807de35d1a569fecbdb240eaaa0ea332372fb"},{"id":"cooperdk/BooruNL-0.8B","revision":"14dbcf2e009f77697dcc7f23e2a706b71b52a5d1"},{"id":"mghafiri/qwen3.5-0.8B-decision-model","revision":"4a9939034a27006b4b62ad5aed65a90b82d0d350"},{"id":"NamanAgnih0tri/AlphaRoute-0.8B-v1.5","revision":"ef87d71f1105f47a15d0d290ee000bcfc0ed68c0"}],"name":"Qwen3.5 0.8B cohort / full ARC-Challenge","parent_task":"sub1b-arc-challenge","priority_models":["Qwen/Qwen3.5-0.8B-Base"],"protocol_version":"codesota-qwen35-arc-challenge-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Challenge/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_qwen35_challenge.py","selection":"All 1172 test items; raw summed conditional answer likelihood; no chat or images; same frozen 14 candidates plus original base control","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":37.615605833008885,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","scope":"8877f1f73ee0a3c138937e07b7f6ca917f2c1d914d223e6a8ffa42b9ac8f0a47","score":0.34215017064846415,"scoring_seconds":32.30435760400724,"seconds":35.97204692498781,"seed":42,"selection":"Full pinned ARC-Challenge test split; all 1172 source-order items","status":"completed","task":"qwen35-arc-challenge-full","text_parameters":752393024,"tf32":false,"truncated_candidates":0,"vision_parameters":100592896,"worker":"white-gpu0"}