{"acc_norm":0.38822525597269625,"acc_norm_definition":"argmax summed conditional likelihood / answer character length, excluding leading delimiter","batch_policy":{"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.26471012202091515},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.04997989803086966},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.05216092604678124},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.05189524102024734},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.05290466290898621},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.05312905099708587},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.05276637407951057},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05289594701025635},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05333731998689473},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05374835594557226},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05332894390448928},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.053093004040420055},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.05815641698427498},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.05839032295625657},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.057858041021972895},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.0590421489905566},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06253162596840411},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06035127001814544},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.060884083970449865},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.05927211698144674},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.05880544800311327},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06054333795327693},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.05912371096201241},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.058807100052945316},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.059414944029413164},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.059284870978444815},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.05912722705397755},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06092469906434417},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.05963118700310588},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.061247951933182776},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.05957793805282563},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06098714598920196},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06073581695090979},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06029355200007558},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06162240996491164},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.060652963002212346},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06660227803513408},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.0667498130351305},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.0651273470139131},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06486857391428202},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06458378303796053},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06495938403531909},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06462190393358469},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06469586293678731},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.0646262529771775},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.0670950060011819},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06779759703204036},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06766022101510316},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06790041795466095},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06798579799942672},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06799383310135454},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06572284793946892},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06597249303013086},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06785793008748442},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06979134492576122},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06852657708805054},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06803766498342156},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06856092100497335},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06790624000132084},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06831265694927424},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0681132759200409},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06701574800536036},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06887550698593259},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06878709211014211},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06974308495409787},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06876431999262422},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06735031201969832},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06954283197410405},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06778650707565248},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06847452907823026},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06911288003902882},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06885877589229494},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06846546195447445},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06875755696091801},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06842430506367236},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07149152597412467},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.0686327739385888},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06981350702699274},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07082188699860126},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07007669704034925},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07203453895635903},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.0731603279709816},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07424483995418996},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07412729097995907},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07292888697702438},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07220984506420791},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07240810408256948},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07288774999324232},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07282209699042141},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07205676008015871},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.0729437539121136},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07478028896730393},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07329517905600369},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.0745994319440797},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07368294207844883},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07703515503089875},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07526688592042774},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07427176996134222},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07254679291509092},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07506201497744769},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07364656403660774},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07794681400991976},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07658294099383056},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.0756671930430457},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.074098828015849},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07373568101320416},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.0756549199577421},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.0751066580414772},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07425724307540804},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07646697503514588},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07526939106173813},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07564813795033842},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07391656807158142},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07431289798114449},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07954801002051681},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07737952598836273},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07731192093342543},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07808468106668442},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07784607610665262},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.08003842493053526},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07875728502403945},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.08038689498789608},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.0786255900748074},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07801093393936753},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07783828198444098},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.08016206498723477},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.08023231604602188},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07879946497268975},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08246037096250802},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.0810682950541377},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08068837691098452},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08118075504899025},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.07940131600480527},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.0791315040551126},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08144124096725136},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08075890806503594},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08114222402218729},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08092670200858265},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08130936499219388},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07950739504303783},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07911929092369974},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08191949303727597},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08698329597245902},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08839253499172628},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08766594005282968},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08938925294205546},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.09173592901788652},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.09005412296392024},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08977398101706058},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08614283904898912},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08831192401703447},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08698650205042213},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08657049702014774},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.09091447794344276},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08897510101087391},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.08911400998476893},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.09168568602763116},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.09092057903762907},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.09021209797356278},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.08857838192488998},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.09252026199828833},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.08941873803269118},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.09242717898450792},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.08913421700708568},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.08765080105513334},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.08778513199649751},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.09050822991412133},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.08993440994527191},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.09371167398057878},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.08891702198889107},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.09169959195423871},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09529233304783702},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09344384504947811},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09315507602877915},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.08804019796662033},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09181020897813141},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09805884095840156},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09510960197076201},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.08988638001028448},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.0895212390460074},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09021096606738865},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09098925802391022},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09893087798263878},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09694176900666207},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09398351109120995},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.09274782706052065},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.08941250701900572},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.08990209992043674},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.08922929503023624},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09916742891073227},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.10127862601075321},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09924730809871107},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09788477595429868},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09936010895762593},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09770616400055587},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.10021794796921313},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.09971937804948539},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.09827953297644854},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.1020777371013537},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.10340927995275706},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.10479088604915887},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.10129042796324939},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.09731751796789467},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.09712716401554644},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.0958699599141255},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.09846726409159601},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.09874775598291308},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.09883375698700547},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.10075310594402254},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.1028644829057157},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.10032181208953261},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.10728719201870263},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.10818631900474429},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.11014644301030785},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.10680889897048473},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10794126195833087},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10896255599800497},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10994028905406594},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.11102136492263526},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10966343304608017},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.1082222149707377},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.11096919805277139},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.11121361400000751},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.11086873104795814},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.1102295289747417},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.11055073793977499},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.11475134105421603},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.11535597906913608},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.11548608203884214},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11574462393764406},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11419846105854958},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11286985198967159},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11582926195114851},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.1179707859409973},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11565500707365572},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11872326896991581},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.12052273494191468},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11932257597800344},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11742785305250436},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11730471299961209},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11765764199662954},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11852100200485438},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11746516195125878},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11661348398774862},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11969420907553285},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11907323205377907},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.11738772795069963},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.11962190503254533},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.12186366505920887},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.12439325300510973},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.12077675899490714},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.11957673099823296},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.12554955994710326},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.12943596707191318},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.12354995007626712},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.13367173704318702},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.13013525097630918},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.12755719397682697},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.1270982880378142},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.13783915794920176},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.13439820101484656},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.12935405399184674},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.12719986704178154},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.1350463000126183},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.1326846859883517},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.13466478907503188},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.13685123599134386},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.13566385197918862},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.13596596498973668},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.15594783902633935},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.15671571099665016},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.15580421104095876},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.1511756801046431},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.15746369608677924},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.15862131502944976},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.17082230898085982},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.1718985759653151},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.17433904798235744},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.17432018206454813},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.17336695396807045},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.17372236808296293},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.177960311062634},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.17698182596359402},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.17965851805638522},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.17481448501348495},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.19101406494155526},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.20543656102381647},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.19723356503527611},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.19792824995238334},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.22227758902590722},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.22303349804133177},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.25004794797860086},{"candidates":15,"kind":"choice","padded_tokens":2685,"seconds":0.3341803480871022}],"chat_template":false,"data_loading_seconds":0.0666221750434488,"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa"},"device":"cuda:0","embedding_head_tied":true,"evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"qwen35-arc-challenge-v1","finished":1792212298,"gpu":1,"id":"4d3599a2d9c1a839e8fb80e2","images_supplied":false,"loaded_class":"Qwen3_5ForCausalLM","loader_ignored_auxiliary_patterns":["^mtp.*","^model.visual.*"],"loading_info":{"error_msgs":[],"mismatched_keys":[],"missing_keys":[],"unexpected_keys":[]},"measured_cost_usd":null,"metric":"accuracy","model":"mithulaartigala/reasoning-summarizer-qwen3.5-0.8b-preview","model_load_seconds":3.076122338999994,"model_revision":"558fc651bf01bbe7cf306ebf2fb32032d7b4e818","n":1172,"notes":"Full raw-accuracy text-only diagnostic for a same-base Qwen3.5 cohort. Stock trusted model classes, no chat or images; generic text scores do not validate specialist OCR/classification/retrieval uses. Unique total/text/vision parameters measured after loading; source-reported metrics are not imported.","optional_kernel_packages":{"causal_conv1d":false,"fla":false,"flash_attn":false,"kernels":false},"parameters":752393024,"peak_gpu_bytes":4111305728,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Challenge / Qwen3.5 same-base cohort","catalogue_id":"arc-challenge","choice_score":"sum_log_likelihood","contract":"Question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"qwen35-arc-challenge-full","license":["cc-by-sa-4.0"],"limit":1172,"metric":"accuracy","models":[{"id":"Qwen/Qwen3.5-0.8B-Base","revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68"},{"id":"Qwen/Qwen3.5-0.8B","revision":"2fc06364715b967f1860aea9cf38778875588b17"},{"id":"Mapika/decider-0.8b","revision":"a0a01d6f8135298f400a8c856b355793012ae971"},{"id":"SeerRay-Lab/Xiaomi-OCR-0","revision":"ff990be318c4c68f7e621dd690231200f47e48c4"},{"id":"unsloth/Qwen3.5-0.8B-Base","revision":"44bc982576bf30f0a9440303598859d92604ea5d"},{"id":"nineninesix/qwen3_5-full-attn-only-14","revision":"a26dff91d90e168da8e49f0c19dde98447c095da"},{"id":"OmniJev/PlayJev-0.8B","revision":"a7348002b1e159add7037d6d50812cd4db2d96e9"},{"id":"mithulaartigala/reasoning-summarizer-qwen3.5-0.8b-preview","revision":"558fc651bf01bbe7cf306ebf2fb32032d7b4e818"},{"id":"sh0ck0r/Qwen3.5-0.8B-heretic","revision":"fb35b60374fc15c99a3890e55a5a77e6d015cf4e"},{"id":"McGill-NLP/AfriqueQwen3.5-0.8B-50Langs","revision":"f49099d79e51bbb783fae622877c4c3415142c82"},{"id":"ahmedeltawel/Qwen3.5-0.8B-hsb-dsb","revision":"7e781a3e970474d3fcb215ec0d3c5832e1131b9f"},{"id":"Kewk/Heretical-Qwen3.5-0.8B","revision":"c16807de35d1a569fecbdb240eaaa0ea332372fb"},{"id":"cooperdk/BooruNL-0.8B","revision":"14dbcf2e009f77697dcc7f23e2a706b71b52a5d1"},{"id":"mghafiri/qwen3.5-0.8B-decision-model","revision":"4a9939034a27006b4b62ad5aed65a90b82d0d350"},{"id":"NamanAgnih0tri/AlphaRoute-0.8B-v1.5","revision":"ef87d71f1105f47a15d0d290ee000bcfc0ed68c0"}],"name":"Qwen3.5 0.8B cohort / full ARC-Challenge","parent_task":"sub1b-arc-challenge","priority_models":["Qwen/Qwen3.5-0.8B-Base"],"protocol_version":"codesota-qwen35-arc-challenge-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Challenge/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_qwen35_challenge.py","selection":"All 1172 test items; raw summed conditional answer likelihood; no chat or images; same frozen 14 candidates plus original base control","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":33.21164660400245,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","scope":"8877f1f73ee0a3c138937e07b7f6ca917f2c1d914d223e6a8ffa42b9ac8f0a47","score":0.34897610921501704,"scoring_seconds":28.219703751034103,"seconds":31.552993295015767,"seed":42,"selection":"Full pinned ARC-Challenge test split; all 1172 source-order items","status":"completed","task":"qwen35-arc-challenge-full","text_parameters":752393024,"tf32":false,"truncated_candidates":0,"vision_parameters":0,"worker":"white-gpu1"}