{"acc_norm":0.27047781569965873,"acc_norm_definition":"argmax summed conditional likelihood / answer character length, excluding leading delimiter","batch_policy":{"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.26374994101934135},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.05161796300671995},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.05544598191045225},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.06060863996390253},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.05647061311174184},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.06159088097047061},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.056439343956299126},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.06215972302015871},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.056864597951062024},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.06274456309620291},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.056824272964149714},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.06219551991671324},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.06100743298884481},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.06962535507045686},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.0769506759243086},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.06884759501554072},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06186065496876836},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06758526095654815},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06961898307781667},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06217222602572292},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06764876004308462},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.0618440939579159},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06829959293827415},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06202299694996327},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.07053978007752448},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.061869731987826526},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06966159294825047},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.0668959750328213},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.073131995042786},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.07005437603220344},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06900995795149356},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.0734646050259471},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06316339399199933},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06966504896990955},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06468748592305928},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06937246292363852},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.0696859989548102},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07685565005522221},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.0752473899628967},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06933102593757212},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07563209801446646},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06856835400685668},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07644204702228308},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.0704041380668059},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07435244100634009},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07746268098708242},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07123700110241771},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07876403897535056},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07161230104975402},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07628442498389632},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07722148206084967},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06912504194770008},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.076330819982104},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.0711025899508968},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.0819581450195983},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.08226583898067474},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07141667592804879},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07729162205941975},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07827738090418279},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07259472296573222},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.08021097700111568},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0701073840027675},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0776187120936811},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07860464998520911},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07195988902822137},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07747896097134799},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07237360090948641},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.08146321203093976},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0801284130429849},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07160167105030268},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07881985197309405},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07926029397640377},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07267798902466893},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07834123994689435},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07484563090838492},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07833744294475764},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07918605499435216},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.0725040229735896},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.0818599519552663},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07957015198189765},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07750635209959},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08353226096369326},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07687329198233783},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08530834305565804},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08451110601890832},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07968628802336752},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08432552102021873},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08462858491111547},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07600675604771823},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08210599096491933},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08394272706937045},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07843538490124047},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08613696799147874},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08862726192455739},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07820976490620524},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08663060900289565},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08695031492970884},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08096587506588548},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08419086900539696},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08668158401269466},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07928167399950325},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08313091204036027},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08857442403677851},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08708560699597001},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07770083600189537},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08642686903476715},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08677795308176428},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07964222598820925},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08516106789465994},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08816332707647234},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08009107306133956},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08662460802588612},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08776411297731102},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07968237099703401},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.09187882707919925},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.09114497795235366},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.08873316005337983},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.08323611796367913},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.08979411795735359},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.09226295293774456},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.0826051210751757},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.08940087503287941},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.09400164498947561},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.08324305096175522},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.09091503906529397},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.0913483970798552},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.09411479707341641},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.08299004996661097},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09683305607177317},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09360277105588466},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.0932967410190031},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08508590795099735},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.0915025039575994},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09211466705892235},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09151696099434048},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08857799007091671},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.09224805503617972},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.09065392205957323},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08953095797915012},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.0917346179485321},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.09260148403700441},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.09352155006490648},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.09478845202829689},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10319893702398986},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10021658497862518},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10392068303190172},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10364988807123154},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10659446998033673},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.0970734239090234},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.10312124202027917},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.09927592100575566},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.10099643003195524},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.10289102303795516},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.0988441159715876},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.10502903000451624},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.10227223997935653},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.10574950301088393},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.10705210396554321},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.10416766395792365},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.0966021450003609},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10533529089298099},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10740764602087438},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.1044879510300234},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10682797594927251},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10679874999914318},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.09476500900927931},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10356827592477202},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10423574096057564},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10457903100177646},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.1021824210183695},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10707350296434015},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09782407304737717},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11129330296535045},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.1085943290963769},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10472721699625254},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10757359606213868},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11379597010090947},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.1090835309587419},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09735622198786587},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10414311802014709},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10124583507422358},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10664021503180265},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.11495396902319044},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10925333900377154},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.11124641494825482},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.10190687805879861},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.10591981105972081},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.10642835008911788},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.10241048701573163},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11312283400911838},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11680070299189538},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11555094295181334},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11552277998998761},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11144770099781454},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.10654200194403529},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11726849596016109},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11711190396454185},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11226310196798295},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11811277002561837},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.12043856794480234},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11870610690675676},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11754951998591423},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11288886796683073},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11173394392244518},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11419108603149652},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11079407099168748},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11609003809280694},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11775919096544385},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11230694293044508},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11712187307421118},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11324895999860018},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1198357529938221},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12559116701595485},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12504428706597537},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12209458602592349},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12783875805325806},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12695085303857923},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12601488700602204},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12291405408177525},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12469492596574128},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12480801600031555},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12645083002280444},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.1274114919360727},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12833470397163182},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12810126796830446},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12412732699885964},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13125461898744106},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.1364797829883173},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.1313895999919623},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.13019784807693213},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1308696500491351},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1297002111095935},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1303736959816888},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13288650196045637},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13249737594742328},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.135745984967798},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13551798998378217},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13456359005067497},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13626888999715447},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13461651804391295},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13311193301342428},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.14197756606154144},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13559085503220558},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13007493899203837},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13586615899112076},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13343381206505},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13411953207105398},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13546445907559246},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13682405604049563},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.141087174997665},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13743600598536432},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13606692396569997},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.14941617904696614},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.1448443429544568},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.13703039893880486},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.153303547995165},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.14487104199361056},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.1447520910296589},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.15185230190400034},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.15546542999800295},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.1467476519756019},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.15036111092194915},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.15365681692492217},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.15249888796824962},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.1523601090302691},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.15785786195192486},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.16493723401799798},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.15384717204142362},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.15708391892258078},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.1823198909405619},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.17432890797499567},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.17990580899640918},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.18038518296089023},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.18048743403051049},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.19060614507179707},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.18916630896274},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.2041767529444769},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.20426107000093907},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.1989355079131201},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.2068145819939673},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.19329277495853603},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.21238683303818107},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.20586171501781791},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.20761841104831547},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.20394508994650096},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.22157543897628784},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.23955530999228358},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.22830070101190358},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.22464637598022819},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.25908211595378816},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.2614409950328991},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.2867600630270317},{"candidates":15,"kind":"choice","padded_tokens":2685,"seconds":0.3844079989939928}],"chat_template":false,"data_loading_seconds":0.05026187398470938,"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa"},"device":"cuda:0","embedding_head_tied":true,"evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"qwen35-arc-challenge-v1","finished":1792212373,"gpu":0,"id":"6ecc59527b7e193dcac98122","images_supplied":false,"loaded_class":"Qwen3_5ForConditionalGeneration","loader_ignored_auxiliary_patterns":["^mtp.*"],"loading_info":{"error_msgs":[],"mismatched_keys":[],"missing_keys":[],"unexpected_keys":[]},"measured_cost_usd":null,"metric":"accuracy","model":"OmniJev/PlayJev-0.8B","model_load_seconds":9.097143913968466,"model_revision":"a7348002b1e159add7037d6d50812cd4db2d96e9","n":1172,"notes":"Full raw-accuracy text-only diagnostic for a same-base Qwen3.5 cohort. Stock trusted model classes, no chat or images; generic text scores do not validate specialist OCR/classification/retrieval uses. Unique total/text/vision parameters measured after loading; source-reported metrics are not imported.","optional_kernel_packages":{"causal_conv1d":false,"fla":false,"flash_attn":false,"kernels":false},"parameters":852985920,"peak_gpu_bytes":4513547264,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Challenge / Qwen3.5 same-base cohort","catalogue_id":"arc-challenge","choice_score":"sum_log_likelihood","contract":"Question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"qwen35-arc-challenge-full","license":["cc-by-sa-4.0"],"limit":1172,"metric":"accuracy","models":[{"id":"Qwen/Qwen3.5-0.8B-Base","revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68"},{"id":"Qwen/Qwen3.5-0.8B","revision":"2fc06364715b967f1860aea9cf38778875588b17"},{"id":"Mapika/decider-0.8b","revision":"a0a01d6f8135298f400a8c856b355793012ae971"},{"id":"SeerRay-Lab/Xiaomi-OCR-0","revision":"ff990be318c4c68f7e621dd690231200f47e48c4"},{"id":"unsloth/Qwen3.5-0.8B-Base","revision":"44bc982576bf30f0a9440303598859d92604ea5d"},{"id":"nineninesix/qwen3_5-full-attn-only-14","revision":"a26dff91d90e168da8e49f0c19dde98447c095da"},{"id":"OmniJev/PlayJev-0.8B","revision":"a7348002b1e159add7037d6d50812cd4db2d96e9"},{"id":"mithulaartigala/reasoning-summarizer-qwen3.5-0.8b-preview","revision":"558fc651bf01bbe7cf306ebf2fb32032d7b4e818"},{"id":"sh0ck0r/Qwen3.5-0.8B-heretic","revision":"fb35b60374fc15c99a3890e55a5a77e6d015cf4e"},{"id":"McGill-NLP/AfriqueQwen3.5-0.8B-50Langs","revision":"f49099d79e51bbb783fae622877c4c3415142c82"},{"id":"ahmedeltawel/Qwen3.5-0.8B-hsb-dsb","revision":"7e781a3e970474d3fcb215ec0d3c5832e1131b9f"},{"id":"Kewk/Heretical-Qwen3.5-0.8B","revision":"c16807de35d1a569fecbdb240eaaa0ea332372fb"},{"id":"cooperdk/BooruNL-0.8B","revision":"14dbcf2e009f77697dcc7f23e2a706b71b52a5d1"},{"id":"mghafiri/qwen3.5-0.8B-decision-model","revision":"4a9939034a27006b4b62ad5aed65a90b82d0d350"},{"id":"NamanAgnih0tri/AlphaRoute-0.8B-v1.5","revision":"ef87d71f1105f47a15d0d290ee000bcfc0ed68c0"}],"name":"Qwen3.5 0.8B cohort / full ARC-Challenge","parent_task":"sub1b-arc-challenge","priority_models":["Qwen/Qwen3.5-0.8B-Base"],"protocol_version":"codesota-qwen35-arc-challenge-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Challenge/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_qwen35_challenge.py","selection":"All 1172 test items; raw summed conditional answer likelihood; no chat or images; same frozen 14 candidates plus original base control","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":43.19695033796597,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","scope":"8877f1f73ee0a3c138937e07b7f6ca917f2c1d914d223e6a8ffa42b9ac8f0a47","score":0.24744027303754265,"scoring_seconds":32.101132826996036,"seconds":41.44727034296375,"seed":42,"selection":"Full pinned ARC-Challenge test split; all 1172 source-order items","status":"completed","task":"qwen35-arc-challenge-full","text_parameters":752393024,"tf32":false,"truncated_candidates":0,"vision_parameters":100592896,"worker":"white-gpu0"}