{"acc_norm":0.3720136518771331,"acc_norm_definition":"argmax summed conditional likelihood / answer character length, excluding leading delimiter","batch_policy":{"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.2762346939416602},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.05031398090068251},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.06058433500584215},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.05560260405763984},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.06192389200441539},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.05588608200196177},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.061999683966860175},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05613751104101539},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.06305160594638437},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05653172603342682},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.062291388981975615},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.0564862119499594},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.06677644303999841},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.06472597806714475},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.08191335096489638},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.0625773819629103},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06842116895131767},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06202894891612232},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06913118402007967},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06213134899735451},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06746064801700413},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06220010796096176},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06779529189225286},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06256081990431994},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06878714193589985},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06787416897714138},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06189015996642411},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.0714647660497576},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06535832793451846},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.075050542014651},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06351539096795022},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.07252089597750455},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06338275398593396},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06838757696095854},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06635442597325891},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.0700862049125135},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07573099201545119},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06993104598950595},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07592888094950467},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06818778405431658},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.0726976350415498},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07089186704251915},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06982006796170026},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07538236200343817},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06864317401777953},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07640189304947853},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07272700895555317},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.0781436109682545},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07766815507784486},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07073971396312118},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07496036496013403},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06968745205085725},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07579721603542566},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07680990395601839},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07434733107220381},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07799890101887286},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07291829702444375},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07400607399176806},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07814440305810422},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07220721000339836},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07859825901687145},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07707375602331012},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07142857799772173},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07768781099002808},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07211163197644055},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07732472394127399},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07634184102062136},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07411246304400265},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.08030769706238061},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07289909094106406},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07961427397094667},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07931874296627939},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07247197395190597},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.0796612820122391},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07829797896556556},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07581135199870914},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.08078789303544909},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07405286200810224},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07927951007150114},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07911735703237355},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07645988103467971},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08495539508294314},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08381008903961629},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07783875206951052},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08503637602552772},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08244328992441297},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07576056709513068},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08464366302359849},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08373851503711194},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07573908707126975},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08397767099086195},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.0844188641058281},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07863469701260328},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.09109847107902169},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08424350793939084},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07950072200037539},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08675957995001227},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08642342209350318},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07721993909217417},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08745743101462722},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.0863222440239042},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08150748501066118},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08853209402877837},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08754251000937074},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07690169499255717},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08638716395944357},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08772001997567713},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08050740894395858},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08579153392929584},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08469157293438911},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08100261399522424},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08893947501201183},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08705152396578342},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.0790000279666856},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.09103144600521773},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.09015262708999217},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.08966890501324087},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.0819134620251134},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.09310471301432699},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.09049059695098549},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.08209631300996989},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.09201728401239961},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.09387082105968148},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.08940809895284474},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.08180351706687361},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.09205276006832719},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.09289744589477777},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.08472956391051412},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09467187500558794},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09311727504245937},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.0924495899816975},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08669308491516858},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09061230404768139},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09328325500246137},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09211979492101818},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08482601295690984},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.09284902503713965},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.09372988797258586},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.09356655296869576},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08638151502236724},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.09424818505067378},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.0932019529864192},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.09810475702397525},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.09749825601466},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.09846108104102314},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10212129796855152},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10396343294996768},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1060875739203766},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.09351687110029161},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.10339013405609876},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.10001922701485455},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.09879814996384084},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.10480404098052531},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.09723345201928169},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.10419515601824969},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.1012347349897027},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.10422180499881506},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.10588566795922816},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.10233372403308749},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.09637997089885175},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10491911496501416},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10810104897245765},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10507186991162598},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10678704897873104},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10323066590353847},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.09351114998571575},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10248037800192833},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10345978394616395},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10351824294775724},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10288690496236086},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10605735902208835},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10070696996990591},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11037077300716192},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10981293197255582},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10121856408659369},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10565736202988774},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11072320898529142},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09919423901010305},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09818340395577252},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10413969098590314},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10496286698617041},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10593109205365181},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.113455704995431},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.11059177399147302},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.0994905709521845},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.10891495703253895},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.10552734893281013},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.10563813697081059},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.10481866798363626},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11233349202666432},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11729954299516976},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11464183800853789},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.10358509700745344},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.1115735350176692},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11531673604622483},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11435128608718514},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11705294391140342},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11206172499805689},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11874441802501678},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11794550891499966},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11515433294698596},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11558507604058832},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11510634399019182},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11086160689592361},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.10379026003647596},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11419040500186384},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11477639793884009},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11778051999863237},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11603681894484907},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.1139571110252291},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11534604092594236},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12142799294088036},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12332245602738112},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12680681399069726},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12077648809645325},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.1284283879213035},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12603214906994253},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12337908102199435},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.1252632859395817},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.1251336440909654},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12641406198963523},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.1250170060666278},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12729471398051828},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12949324399232864},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.1263538389466703},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.125285457004793},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.1306928809499368},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13338123506400734},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13015002803876996},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1301978980191052},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.130607643048279},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.13083313289098442},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1310262029292062},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13875378400553018},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.1347564689349383},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13574818801134825},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13300040399190038},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13545065408106893},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.1346389000536874},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13465182401705533},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.1331932240864262},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13629419705830514},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.134391137980856},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13167084497399628},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.1345882760360837},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.14383636298589408},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13319805299397558},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.1356399670476094},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13413744501303881},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.14242220507003367},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13872298609931022},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13471335894428194},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.14004575298167765},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.1445483110146597},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.14681858394760638},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.1500575039535761},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.1448373389430344},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.1440563229843974},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.1444815960712731},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.16515595198143274},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.14254773897118866},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.14761232398450375},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.14890389295760542},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.1528313970193267},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.16186687699519098},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.15421921596862376},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.1589317440520972},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.15353144297841936},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.16404904704540968},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.17462538194376975},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.18298230599611998},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.17792280099820346},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.17455241491552442},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.18598065804690123},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.18086162093095481},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.19832676299847662},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.20585167594254017},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.19437951094005257},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.20458337990567088},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.19934734608978033},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.20169706689193845},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.21010592998936772},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.19645721605047584},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.21167879295535386},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.20893134898506105},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.21383039595093578},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.23707607702817768},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.22565433697309345},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.23511839599814266},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.25885455100797117},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.2553663069847971},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.2869671389926225},{"candidates":15,"kind":"choice","padded_tokens":2685,"seconds":0.3927604149794206}],"chat_template":false,"data_loading_seconds":0.0633056799415499,"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa"},"device":"cuda:0","embedding_head_tied":true,"evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"qwen35-arc-challenge-v1","finished":1792212205,"gpu":0,"id":"38365bef2439fd792f653a37","images_supplied":false,"loaded_class":"Qwen3_5ForConditionalGeneration","loader_ignored_auxiliary_patterns":["^mtp.*"],"loading_info":{"error_msgs":[],"mismatched_keys":[],"missing_keys":[],"unexpected_keys":[]},"measured_cost_usd":null,"metric":"accuracy","model":"sh0ck0r/Qwen3.5-0.8B-heretic","model_load_seconds":7.560219279956073,"model_revision":"fb35b60374fc15c99a3890e55a5a77e6d015cf4e","n":1172,"notes":"Full raw-accuracy text-only diagnostic for a same-base Qwen3.5 cohort. Stock trusted model classes, no chat or images; generic text scores do not validate specialist OCR/classification/retrieval uses. Unique total/text/vision parameters measured after loading; source-reported metrics are not imported.","optional_kernel_packages":{"causal_conv1d":false,"fla":false,"flash_attn":false,"kernels":false},"parameters":852985920,"peak_gpu_bytes":4513547264,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Challenge / Qwen3.5 same-base cohort","catalogue_id":"arc-challenge","choice_score":"sum_log_likelihood","contract":"Question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"qwen35-arc-challenge-full","license":["cc-by-sa-4.0"],"limit":1172,"metric":"accuracy","models":[{"id":"Qwen/Qwen3.5-0.8B-Base","revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68"},{"id":"Qwen/Qwen3.5-0.8B","revision":"2fc06364715b967f1860aea9cf38778875588b17"},{"id":"Mapika/decider-0.8b","revision":"a0a01d6f8135298f400a8c856b355793012ae971"},{"id":"SeerRay-Lab/Xiaomi-OCR-0","revision":"ff990be318c4c68f7e621dd690231200f47e48c4"},{"id":"unsloth/Qwen3.5-0.8B-Base","revision":"44bc982576bf30f0a9440303598859d92604ea5d"},{"id":"nineninesix/qwen3_5-full-attn-only-14","revision":"a26dff91d90e168da8e49f0c19dde98447c095da"},{"id":"OmniJev/PlayJev-0.8B","revision":"a7348002b1e159add7037d6d50812cd4db2d96e9"},{"id":"mithulaartigala/reasoning-summarizer-qwen3.5-0.8b-preview","revision":"558fc651bf01bbe7cf306ebf2fb32032d7b4e818"},{"id":"sh0ck0r/Qwen3.5-0.8B-heretic","revision":"fb35b60374fc15c99a3890e55a5a77e6d015cf4e"},{"id":"McGill-NLP/AfriqueQwen3.5-0.8B-50Langs","revision":"f49099d79e51bbb783fae622877c4c3415142c82"},{"id":"ahmedeltawel/Qwen3.5-0.8B-hsb-dsb","revision":"7e781a3e970474d3fcb215ec0d3c5832e1131b9f"},{"id":"Kewk/Heretical-Qwen3.5-0.8B","revision":"c16807de35d1a569fecbdb240eaaa0ea332372fb"},{"id":"cooperdk/BooruNL-0.8B","revision":"14dbcf2e009f77697dcc7f23e2a706b71b52a5d1"},{"id":"mghafiri/qwen3.5-0.8B-decision-model","revision":"4a9939034a27006b4b62ad5aed65a90b82d0d350"},{"id":"NamanAgnih0tri/AlphaRoute-0.8B-v1.5","revision":"ef87d71f1105f47a15d0d290ee000bcfc0ed68c0"}],"name":"Qwen3.5 0.8B cohort / full ARC-Challenge","parent_task":"sub1b-arc-challenge","priority_models":["Qwen/Qwen3.5-0.8B-Base"],"protocol_version":"codesota-qwen35-arc-challenge-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Challenge/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_qwen35_challenge.py","selection":"All 1172 test items; raw summed conditional answer likelihood; no chat or images; same frozen 14 candidates plus original base control","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":41.54360213293694,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","scope":"8877f1f73ee0a3c138937e07b7f6ca917f2c1d914d223e6a8ffa42b9ac8f0a47","score":0.32849829351535836,"scoring_seconds":32.03479050309397,"seconds":39.85693414998241,"seed":42,"selection":"Full pinned ARC-Challenge test split; all 1172 source-order items","status":"completed","task":"qwen35-arc-challenge-full","text_parameters":752393024,"tf32":false,"truncated_candidates":0,"vision_parameters":100592896,"worker":"white-gpu0"}