{"acc_norm":0.3924914675767918,"acc_norm_definition":"argmax summed conditional likelihood / answer character length, excluding leading delimiter","batch_policy":{"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.27059666998684406},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.05106800701469183},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.05164428206626326},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.05150465189944953},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.052748361020348966},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.052898191031999886},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.05259162897709757},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.052761917002499104},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.053082414087839425},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05285111197736114},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.053171649924479425},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05293566093314439},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.05681369302328676},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.05770532600581646},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.05761552799958736},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.05912647594232112},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06185642699711025},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.0596004999242723},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06086134200450033},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.05902604793664068},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.058476262958720326},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.0579428980126977},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.058534151990897954},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.058339087991043925},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.05834013898856938},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.058445605100132525},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.05860346998088062},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.05994055396877229},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.058623527991585433},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06054978002794087},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.059392913011834025},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06212096998933703},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06096126697957516},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.060208193957805634},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06197186198551208},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06066413305234164},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06579367897938937},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06672775105107576},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06422354094684124},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06411742290947586},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.0639641679590568},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.0644268310861662},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.0637193110305816},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.0638575489865616},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06391702906694263},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06653484201524407},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06769904401153326},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06788529991172254},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06819877598900348},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06799802999012172},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06683829706162214},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06568560900632292},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06475541391409934},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.0671441680751741},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06915405602194369},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06732601707335562},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06743603304494172},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06737362593412399},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06714904704131186},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06776546710170805},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06787986098788679},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06663163297343999},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0679993029916659},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06965791597031057},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06848924700170755},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06835827301256359},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06621865194756538},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06908349494915456},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06731835298705846},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06786677602212876},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.0686795610236004},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06802238593809307},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06728409905917943},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06721316697075963},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06784521602094173},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06880488502793014},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.0686929669464007},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06934155605267733},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.069950653007254},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.0701727969571948},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07100138207897544},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07189226301852614},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07109979609958827},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.0733591279713437},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07365502999164164},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.0713209779933095},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07119212800171226},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07099768507760018},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07094070792663842},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07198822207283229},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07232611195649952},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07289748801849782},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07369578594807535},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07402175397146493},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07241292402613908},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07450889307074249},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.0742573729949072},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07383536593988538},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.072377217002213},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07430374994874},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07267915003467351},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07680590695235878},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07808016298804432},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07610360707622021},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07164536300115287},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07252560299821198},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07441185205243528},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07490180607419461},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07328577199950814},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.0732290759915486},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07678989705163985},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07507975702174008},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07333457306958735},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07272496598307043},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07956761703826487},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07903860101941973},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07712380902376026},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07691129401791841},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07655929704196751},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07878455601166934},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07706935796886683},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07870365597773343},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07841198204550892},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.0777073169592768},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07685295399278402},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07990193006116897},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07951828499790281},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07830554398242384},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08236734801903367},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.07989257294684649},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.0792457569623366},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.07990065799094737},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.07854869705624878},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.07820133899804205},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08029991202056408},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08017775404732674},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07847485889215022},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07884736405685544},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08140912093222141},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07956055400427431},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07873913203366101},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08011324505787343},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08607519301585853},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08777017402462661},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08662557904608548},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08800629409961402},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08765153307467699},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.09023401897866279},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08881566405761987},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08590393303893507},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08532788907177746},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08624015096575022},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08954521489795297},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08869184297509491},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.09057939308695495},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.08695079595781863},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.09234896302223206},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.08875898807309568},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.08840865502133965},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.08767022797837853},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.09188501804601401},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.08970052294898778},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.09112334693782032},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.08841470605693758},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.09028793894685805},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.08658475300762802},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.09086154901888222},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.08866510295774788},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.0919017189880833},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.08705931901931763},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.09354602603707463},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09001659299246967},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09256978495977819},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09597187000326812},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.08751254400704056},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09140224708244205},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.0966011630371213},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09471286193002015},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.08823454100638628},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.08864655799698085},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.08881497196853161},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09025434695649892},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09576694702263921},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09783661703113467},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.0954251199727878},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.09309945302084088},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.08883914700709283},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.0920064749661833},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.08786656404845417},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.10005851101595908},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.099880798952654},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09840671997517347},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09715469507500529},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09823015006259084},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09667762508615851},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.0997965820133686},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.10031881602481008},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.09738416306208819},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.10231975803617388},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.10292110894806683},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.10320532007608563},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.1005516400327906},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.09540124493651092},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.0982563899597153},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.09464748902246356},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.09553343208972365},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.09812005492858589},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.0984739059349522},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.10175138805061579},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.09997829108033329},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.09819752001203597},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1052711020456627},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.10613538301549852},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.10755803692154586},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.10277778306044638},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10649134695995599},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10897110297810286},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10799414000939578},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10925401898566633},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10830244503449649},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.10626045695971698},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.10949739406351},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.10716193809639663},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.10840158991049975},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.10910697595681995},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.10700251103844494},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.11165483691729605},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.11399651400279254},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.11039554805029184},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11235262802802026},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11000871693249792},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.10910203598905355},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11111806496046484},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11246656998991966},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11316126701422036},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11713591904845089},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11670094798319042},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11532574205193669},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11367127695120871},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11481480998918414},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11273924901615828},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11477448500227183},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11509328905958682},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.1156504579121247},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11475051997695118},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11430264404043555},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.1139454689109698},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.11692225001752377},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.1182519500143826},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.12120113999117166},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.12104402692057192},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.1168976139742881},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.12062163895461708},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.12534311402123421},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.11796433397103101},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.12969477905426174},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.1257123629329726},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.12688257498666644},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.12442556291352957},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.13395069702528417},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.12857224605977535},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.12749578896909952},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.12517152400687337},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.13264502095989883},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.13338456093333662},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.1359472799813375},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.1357280210359022},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.1341211840044707},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.13273069204296917},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.1572286980226636},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.15473020891658962},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.15431449399329722},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.1505663039861247},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.15540139004588127},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.15808602701872587},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.16857246297877282},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.17035056999884546},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.17046838894020766},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.17359549005050212},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.17317780200392008},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.16996521095279604},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.17791437497362494},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.17329060204792768},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.17827622999902815},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.17350270797032863},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.18786743702366948},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.20379582198802382},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.1942294609034434},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.19579955888912082},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.2216236590174958},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.21956568106543273},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.2486875409958884},{"candidates":15,"kind":"choice","padded_tokens":2685,"seconds":0.32814604393206537}],"chat_template":false,"data_loading_seconds":0.06594071397557855,"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa"},"device":"cuda:0","embedding_head_tied":true,"evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"qwen35-arc-challenge-v1","finished":1792212200,"gpu":1,"id":"3a8d7c60e2066a56464fef3a","images_supplied":false,"loaded_class":"Qwen3_5ForCausalLM","loader_ignored_auxiliary_patterns":["^mtp.*","^model.visual.*"],"loading_info":{"error_msgs":[],"mismatched_keys":[],"missing_keys":[],"unexpected_keys":[]},"measured_cost_usd":null,"metric":"accuracy","model":"NamanAgnih0tri/AlphaRoute-0.8B-v1.5","model_load_seconds":3.2469474331010133,"model_revision":"ef87d71f1105f47a15d0d290ee000bcfc0ed68c0","n":1172,"notes":"Full raw-accuracy text-only diagnostic for a same-base Qwen3.5 cohort. Stock trusted model classes, no chat or images; generic text scores do not validate specialist OCR/classification/retrieval uses. Unique total/text/vision parameters measured after loading; source-reported metrics are not imported.","optional_kernel_packages":{"causal_conv1d":false,"fla":false,"flash_attn":false,"kernels":false},"parameters":752393024,"peak_gpu_bytes":4111305728,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Challenge / Qwen3.5 same-base cohort","catalogue_id":"arc-challenge","choice_score":"sum_log_likelihood","contract":"Question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"qwen35-arc-challenge-full","license":["cc-by-sa-4.0"],"limit":1172,"metric":"accuracy","models":[{"id":"Qwen/Qwen3.5-0.8B-Base","revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68"},{"id":"Qwen/Qwen3.5-0.8B","revision":"2fc06364715b967f1860aea9cf38778875588b17"},{"id":"Mapika/decider-0.8b","revision":"a0a01d6f8135298f400a8c856b355793012ae971"},{"id":"SeerRay-Lab/Xiaomi-OCR-0","revision":"ff990be318c4c68f7e621dd690231200f47e48c4"},{"id":"unsloth/Qwen3.5-0.8B-Base","revision":"44bc982576bf30f0a9440303598859d92604ea5d"},{"id":"nineninesix/qwen3_5-full-attn-only-14","revision":"a26dff91d90e168da8e49f0c19dde98447c095da"},{"id":"OmniJev/PlayJev-0.8B","revision":"a7348002b1e159add7037d6d50812cd4db2d96e9"},{"id":"mithulaartigala/reasoning-summarizer-qwen3.5-0.8b-preview","revision":"558fc651bf01bbe7cf306ebf2fb32032d7b4e818"},{"id":"sh0ck0r/Qwen3.5-0.8B-heretic","revision":"fb35b60374fc15c99a3890e55a5a77e6d015cf4e"},{"id":"McGill-NLP/AfriqueQwen3.5-0.8B-50Langs","revision":"f49099d79e51bbb783fae622877c4c3415142c82"},{"id":"ahmedeltawel/Qwen3.5-0.8B-hsb-dsb","revision":"7e781a3e970474d3fcb215ec0d3c5832e1131b9f"},{"id":"Kewk/Heretical-Qwen3.5-0.8B","revision":"c16807de35d1a569fecbdb240eaaa0ea332372fb"},{"id":"cooperdk/BooruNL-0.8B","revision":"14dbcf2e009f77697dcc7f23e2a706b71b52a5d1"},{"id":"mghafiri/qwen3.5-0.8B-decision-model","revision":"4a9939034a27006b4b62ad5aed65a90b82d0d350"},{"id":"NamanAgnih0tri/AlphaRoute-0.8B-v1.5","revision":"ef87d71f1105f47a15d0d290ee000bcfc0ed68c0"}],"name":"Qwen3.5 0.8B cohort / full ARC-Challenge","parent_task":"sub1b-arc-challenge","priority_models":["Qwen/Qwen3.5-0.8B-Base"],"protocol_version":"codesota-qwen35-arc-challenge-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Challenge/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_qwen35_challenge.py","selection":"All 1172 test items; raw summed conditional answer likelihood; no chat or images; same frozen 14 candidates plus original base control","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":33.11008321400732,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","scope":"8877f1f73ee0a3c138937e07b7f6ca917f2c1d914d223e6a8ffa42b9ac8f0a47","score":0.3609215017064846,"scoring_seconds":27.89761999004986,"seconds":31.42348099395167,"seed":42,"selection":"Full pinned ARC-Challenge test split; all 1172 source-order items","status":"completed","task":"qwen35-arc-challenge-full","text_parameters":752393024,"tf32":false,"truncated_candidates":0,"vision_parameters":0,"worker":"white-gpu1"}