{"acc_norm":0.4044368600682594,"acc_norm_definition":"argmax summed conditional likelihood / answer character length, excluding leading delimiter","batch_policy":{"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.2594349649734795},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.056537106982432306},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.05620824405923486},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.056325983023270965},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.06200511404313147},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.056788504938595},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.06141701794695109},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05714092298876494},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.06210763496346772},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.057284009992145},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.06298796692863107},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05696572596207261},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.06670203397516161},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.061739758937619627},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.07465445401612669},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.06267527397722006},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06765767594333738},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06205393502023071},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.0628286199644208},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06749628391116858},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06113532301969826},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06853194604627788},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.0627026850124821},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06891221599653363},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06199218099936843},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06910585600417107},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.062427812023088336},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.07044183905236423},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.0655127459904179},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.0708522730274126},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.0663608469767496},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06957171589601785},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06919940107036382},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06330905703362077},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.07062976900488138},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06254322698805481},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07550919789355248},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07083158497698605},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.0752617260441184},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06815702794119716},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07469227502588183},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07510280993301421},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06865389400627464},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07671123999170959},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.0690944850211963},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.0776583559345454},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07862264499999583},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06987844698596746},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07613044697791338},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07131585699971765},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.0771853739861399},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07508506707381457},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06825650203973055},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.0771541849244386},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07585654605645686},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07915111002512276},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0779535569017753},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07245258800685406},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07891013100743294},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07237157702911645},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07976043701637536},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07645453093573451},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07178493193350732},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07909945398569107},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07727293705102056},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07329973811283708},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07800322992261499},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07349469093605876},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0767636380624026},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07883021200541407},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07216122501995414},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.08316433394793421},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07216667500324547},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07757220510393381},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07974747300613672},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07197028899099678},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.08028566592838615},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07870525808539242},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07613733899779618},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.08017330698203295},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07591621694155037},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08505034202244133},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08220783004071563},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07872412400320172},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08344822400249541},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.0821209290297702},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07656523701734841},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08353134896606207},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08291089197155088},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.0775739389937371},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08412991499062628},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08462253399193287},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07771727594081312},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08583962405100465},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08593274699524045},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07869359699543566},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08890753507148474},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08413897210266441},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07726858905516565},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08622460206970572},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08654347597621381},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08235431392677128},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08971434994600713},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08724565606098622},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07724425406195223},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.0836739749647677},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08855892496649176},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08006141800433397},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08386233600322157},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08799879997968674},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08584458299446851},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.0796335400082171},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08758588996715844},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08480261999648064},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.0846260900143534},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.08933009300380945},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.08778450102545321},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.08199243003036827},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.08920194499660283},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.0909950080094859},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.0838615040993318},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.09375837096013129},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.08876392699312419},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.0894046020694077},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.0819662701105699},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.09357755503151566},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.0919603779911995},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.0822922179941088},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09238098305650055},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09682366799097508},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09174916404299438},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08632322610355914},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09007036394905299},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08961270097643137},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09251453203614801},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08584766800049692},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.09191213897429407},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08944172190967947},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08820514508988708},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.09009625192265958},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.0931954609695822},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.09334497002419084},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.09282508096657693},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10360503499396145},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.09816716401837766},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10179668199270964},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10319301602430642},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.095330965006724},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.09984257793985307},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.10117291903588921},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.09931237006094307},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.10176794801373035},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.09703663596883416},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.10494194796774536},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.10486572608351707},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.10028031398542225},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.10588414501398802},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.10295186599250883},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.09322921489365399},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10231563006527722},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10421914991457015},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10522624803707004},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10544291196856648},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.1070799850858748},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.09652561298571527},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10071327199693769},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10094334103632718},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10335767304059118},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10467231494840235},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.09310587495565414},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10535511805210263},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10718780709430575},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10906205198261887},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10994891601148993},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10276940697804093},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10528306395281106},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10338332096580416},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10919947794172913},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09964446804951876},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10372476803604513},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1045884289778769},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10475708299782127},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10762500099372119},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10765052901115268},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10907040804158896},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.10784699500072747},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.10414792702067643},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.10438638203777373},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.10156700399238616},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.10651670407969505},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11471115704625845},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11341729399282485},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.1111520390259102},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11183876998256892},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11383283801842481},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11720603995490819},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11578182398807257},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11230350704863667},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11476838309317827},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.1147593860514462},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11068952607456595},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11660739406943321},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11155431007500738},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11169714597053826},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11066172400023788},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11066327791195363},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11223544902168214},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11592036101501435},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11486773903016001},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11654706997796893},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11209094000514597},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12437974708154798},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12377498007845134},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.11759091704152524},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.11538503307383507},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12393597001209855},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12389873003121465},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12517893803305924},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12515387195162475},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12649795797187835},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12257720599882305},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12616910389624536},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.13443718408234417},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.1237474090885371},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12508904107380658},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12395410402677953},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13162125297822058},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.1297012410359457},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13073415902908891},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1304564509773627},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.12757800100371242},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1291955290362239},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1306786749046296},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13178634003270417},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13062257098499686},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13432019599713385},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.1386727320495993},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.1353609369834885},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.1315822700271383},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13143501500599086},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13459226302802563},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.1396960309939459},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13375088397879153},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13510355702601373},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13402039697393775},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13375718600582331},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13075018802192062},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.1356691310647875},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.1356305390363559},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.1392011680873111},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.14040138700511307},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13773873099125922},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.14554523001424968},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.1446785629959777},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.1387706659734249},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.14958044490776956},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.14665775396861136},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.14593598898500204},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.15132457797881216},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.15309336606878787},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.14721153699792922},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.14847820904105902},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.14679891802370548},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.16218689503148198},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.1511223199777305},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.15486050106119365},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.15872694202698767},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.16087275405880064},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.15301339700818062},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.1765386590268463},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.1839893040014431},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.17560885497368872},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.1839696979150176},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.17533549608197063},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.18820048798806965},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.1911355609772727},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.2012171340174973},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.19792975194286555},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.2043392350897193},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.2045234190300107},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.1942544379271567},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.2068661079974845},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.20074325904715806},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.20319838798604906},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.2052829050226137},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.22236119501758367},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.2304527349770069},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.22751312295440584},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.23031777294818312},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.25776792492251843},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.25350106798578054},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.29425746598280966},{"candidates":15,"kind":"choice","padded_tokens":2685,"seconds":0.38478076399769634}],"chat_template":false,"data_loading_seconds":0.06125318200793117,"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa"},"device":"cuda:0","embedding_head_tied":true,"evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"qwen35-arc-challenge-v1","finished":1792212045,"gpu":0,"id":"584079da775396492e0ed153","images_supplied":false,"loaded_class":"Qwen3_5ForConditionalGeneration","loader_ignored_auxiliary_patterns":["^mtp.*"],"loading_info":{"error_msgs":[],"mismatched_keys":[],"missing_keys":[],"unexpected_keys":[]},"measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen3.5-0.8B-Base","model_load_seconds":2.4780071830609813,"model_revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68","n":1172,"notes":"Full raw-accuracy text-only diagnostic for a same-base Qwen3.5 cohort. Stock trusted model classes, no chat or images; generic text scores do not validate specialist OCR/classification/retrieval uses. Unique total/text/vision parameters measured after loading; source-reported metrics are not imported.","optional_kernel_packages":{"causal_conv1d":false,"fla":false,"flash_attn":false,"kernels":false},"parameters":852985920,"peak_gpu_bytes":4513547264,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Challenge / Qwen3.5 same-base cohort","catalogue_id":"arc-challenge","choice_score":"sum_log_likelihood","contract":"Question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"qwen35-arc-challenge-full","license":["cc-by-sa-4.0"],"limit":1172,"metric":"accuracy","models":[{"id":"Qwen/Qwen3.5-0.8B-Base","revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68"},{"id":"Qwen/Qwen3.5-0.8B","revision":"2fc06364715b967f1860aea9cf38778875588b17"},{"id":"Mapika/decider-0.8b","revision":"a0a01d6f8135298f400a8c856b355793012ae971"},{"id":"SeerRay-Lab/Xiaomi-OCR-0","revision":"ff990be318c4c68f7e621dd690231200f47e48c4"},{"id":"unsloth/Qwen3.5-0.8B-Base","revision":"44bc982576bf30f0a9440303598859d92604ea5d"},{"id":"nineninesix/qwen3_5-full-attn-only-14","revision":"a26dff91d90e168da8e49f0c19dde98447c095da"},{"id":"OmniJev/PlayJev-0.8B","revision":"a7348002b1e159add7037d6d50812cd4db2d96e9"},{"id":"mithulaartigala/reasoning-summarizer-qwen3.5-0.8b-preview","revision":"558fc651bf01bbe7cf306ebf2fb32032d7b4e818"},{"id":"sh0ck0r/Qwen3.5-0.8B-heretic","revision":"fb35b60374fc15c99a3890e55a5a77e6d015cf4e"},{"id":"McGill-NLP/AfriqueQwen3.5-0.8B-50Langs","revision":"f49099d79e51bbb783fae622877c4c3415142c82"},{"id":"ahmedeltawel/Qwen3.5-0.8B-hsb-dsb","revision":"7e781a3e970474d3fcb215ec0d3c5832e1131b9f"},{"id":"Kewk/Heretical-Qwen3.5-0.8B","revision":"c16807de35d1a569fecbdb240eaaa0ea332372fb"},{"id":"cooperdk/BooruNL-0.8B","revision":"14dbcf2e009f77697dcc7f23e2a706b71b52a5d1"},{"id":"mghafiri/qwen3.5-0.8B-decision-model","revision":"4a9939034a27006b4b62ad5aed65a90b82d0d350"},{"id":"NamanAgnih0tri/AlphaRoute-0.8B-v1.5","revision":"ef87d71f1105f47a15d0d290ee000bcfc0ed68c0"}],"name":"Qwen3.5 0.8B cohort / full ARC-Challenge","parent_task":"sub1b-arc-challenge","priority_models":["Qwen/Qwen3.5-0.8B-Base"],"protocol_version":"codesota-qwen35-arc-challenge-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Challenge/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_qwen35_challenge.py","selection":"All 1172 test items; raw summed conditional answer likelihood; no chat or images; same frozen 14 candidates plus original base control","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":36.270080911926925,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","scope":"8877f1f73ee0a3c138937e07b7f6ca917f2c1d914d223e6a8ffa42b9ac8f0a47","score":0.3848122866894198,"scoring_seconds":31.8661073949188,"seconds":34.59673888096586,"seed":42,"selection":"Full pinned ARC-Challenge test split; all 1172 source-order items","status":"completed","task":"qwen35-arc-challenge-full","text_parameters":752393024,"tf32":false,"truncated_candidates":0,"vision_parameters":100592896,"worker":"white-gpu0"}