{"acc_norm":0.2593856655290102,"acc_norm_definition":"argmax summed conditional likelihood / answer character length, excluding leading delimiter","batch_policy":{"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.1610426310217008},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.015877270023338497},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.01735624799039215},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.023190718959085643},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.025974991032853723},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.021739813033491373},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.01946646301075816},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.020480394014157355},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.02056238695513457},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.02064386906567961},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.02617708791512996},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.0204642741009593},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.022507134010083973},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.024489570991136134},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.02398153394460678},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.029593258048407733},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.02297453593928367},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.023963068961165845},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.02559931098949164},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.02406368707306683},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.03020138305146247},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.02303342497907579},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.023641469073481858},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.024112347047775984},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.02516713400837034},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.030377462040632963},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.02355836401693523},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.027434793999418616},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.02367681509349495},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.026454715989530087},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.0257381999399513},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.025772823952138424},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.026499238912947476},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.028531459043733776},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.04293665289878845},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.027390050003305078},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.029287207988090813},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.029770669993013144},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.032884457032196224},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.026980716036632657},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.026807263027876616},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.026955440058372915},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.034262937027961016},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.02619749598670751},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.027297256980091333},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.028235666919499636},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.03749000502284616},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.029238627990707755},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.029378958977758884},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.029440163052640855},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.033435394056141376},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.027533298009075224},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.02700666501186788},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.029360795044340193},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.03712399397045374},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.02949758106842637},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.02830818307120353},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03195398091338575},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03742919198703021},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.029466612031683326},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.030764183029532433},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.028116664965637028},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03692891902755946},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03001609700731933},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03164852107875049},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03350835002493113},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0371345030143857},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.031836702954024076},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.02924297598656267},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.030747681972570717},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.03617263899650425},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.03034101298544556},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.030839171959087253},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.03667353291530162},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.03177867399062961},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.030078332987613976},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.030638248077593744},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.03735476394649595},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.031872099032625556},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.030940552009269595},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.03274276200681925},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.03842425695620477},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.03150967299006879},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.035069470992311835},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.04058923595584929},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.03162257303483784},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.031761532998643816},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.03256919898558408},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.038397197029553354},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.029777031973935664},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.03292708599474281},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.03586576599627733},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.04059496591798961},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.034609592985361814},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.03407662990503013},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.04202614596579224},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.03473221301101148},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.03627566003706306},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.04006145091261715},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.03596835792995989},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.03513964300509542},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.037519080098718405},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.04359444999136031},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.034520779037848115},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.03096238197758794},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.03908263496123254},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.03281577804591507},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.033574883942492306},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.033581256051547825},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.04072145198006183},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.03493334702216089},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.03821258305106312},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.041544065927155316},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.032688400940969586},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.03654189791996032},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.04093764501158148},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.034667112049646676},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.03484873007982969},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.03614070895127952},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.04606642003636807},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.0347842900082469},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.03725613991264254},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.04493284597992897},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.03533252200577408},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.03836080990731716},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.04581107303965837},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.03693227597977966},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.03766742697916925},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.04617763694841415},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.03576097101904452},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.037858503055758774},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.04498191806487739},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.03500381007324904},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.036550082033500075},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.04607464501168579},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.0357256350107491},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.03917885501869023},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.04345125309191644},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.038359147030860186},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.036461047013290226},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.042704480001702905},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.040996304946020246},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.04154822404962033},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.05052574595902115},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.040183779085054994},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.04767545103095472},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.053507567034102976},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.04612823505885899},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.05200624803546816},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.04254753689747304},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.041564884944818914},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.05049522896297276},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.04174646397586912},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.04765382001642138},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.050482064951211214},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.04343125596642494},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.052852645982056856},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.042495471076108515},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.0467087869765237},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.050753511022776365},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.043854885967448354},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.0482072529848665},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.05322143400553614},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.044318270054645836},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.05194269900675863},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.040352024021558464},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.04400530597195029},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.05311484495177865},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.04450434702448547},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.04408600693568587},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.053290141047909856},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.04580366995651275},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.057249084929935634},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.04717992607038468},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.04383712203707546},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.054894603905268013},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.051904547959566116},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.054268666077405214},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.040122375008650124},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.043911600951105356},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.0506885489448905},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.04473773203790188},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.061546007986180484},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.0439021730562672},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.04897330200765282},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.05400236998684704},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.0441957819275558},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.05241410806775093},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.04139279399532825},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.049480566987767816},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.05479949607979506},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.049411418032832444},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.05516800296027213},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.04593062691856176},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.0486894219648093},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.05673458508681506},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.05230566603131592},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.054364276002161205},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.05031733703799546},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.059510952909477055},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.04882242891471833},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.0492921459954232},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.05198548804037273},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.04479908698704094},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.05080960597842932},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.044622015906497836},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.04806789301801473},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.05619074101559818},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.04978217906318605},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.057230410049669445},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.04477042402140796},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.05943788692820817},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.053646336076781154},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.05719735799357295},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.05937179294414818},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.05456236400641501},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.05872039799578488},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.05401657603215426},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.0612639719620347},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.05329283594619483},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.05942390998825431},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.05176264303736389},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.06413764099124819},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.052235514973290265},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.06197783397510648},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.05053861008491367},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.05790628993418068},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.061586263007484376},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.058304322068579495},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.06196148204617202},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.05683174601290375},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.06366657302714884},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.05493933707475662},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.06265300302766263},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.05707228393293917},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.06656158098485321},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.061112659983336926},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.06526163790840656},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.05852233897894621},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.06401021406054497},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.05754787300247699},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.0649106219643727},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.058287080959416926},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.06554216099902987},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.058092338033020496},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.06613576703239232},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.057278690044768155},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.06583641900215298},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.059811573009938},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.070091164088808},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.062465171911753714},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.06525348301511258},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.06562357291113585},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.07179861795157194},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.07044725795276463},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.07131758192554116},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.07192154694348574},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.06356668600346893},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.07297907001338899},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.07678847399074584},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.0721627480816096},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.07789545797277242},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.06333449296653271},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.07430839899461716},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.06649219198152423},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.07718230900354683},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.06853741605300456},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.07442242093384266},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.077210110030137},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.06771170697174966},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.07900069002062082},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.0738803599961102},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.07138371502514929},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.07698877702932805},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.07016036403365433},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.08440630102995783},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.089253720943816},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.08000505203381181},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.08666624606121331},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.08920960896648467},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.07677963795140386},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.09090553096029907},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.09009005990810692},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.0858162899967283},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.08334686397574842},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.09145177993923426},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.11119428696110845},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.08605198003351688},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.09877803293056786},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.12185932695865631},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.11267287598457187},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.1233392080757767},{"candidates":15,"kind":"choice","padded_tokens":2685,"seconds":0.16584624897222966}],"chat_template":false,"data_loading_seconds":0.04891479201614857,"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa"},"device":"cuda:0","embedding_head_tied":true,"evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"qwen35-arc-challenge-v1","finished":1792212147,"gpu":0,"id":"1b595b2b874b0d615aa89161","images_supplied":false,"loaded_class":"Qwen3_5ForCausalLM","loader_ignored_auxiliary_patterns":["^mtp.*","^model.visual.*"],"loading_info":{"error_msgs":[],"mismatched_keys":[],"missing_keys":[],"unexpected_keys":[]},"measured_cost_usd":null,"metric":"accuracy","model":"nineninesix/qwen3_5-full-attn-only-14","model_load_seconds":3.1047331349691376,"model_revision":"a26dff91d90e168da8e49f0c19dde98447c095da","n":1172,"notes":"Full raw-accuracy text-only diagnostic for a same-base Qwen3.5 cohort. Stock trusted model classes, no chat or images; generic text scores do not validate specialist OCR/classification/retrieval uses. Unique total/text/vision parameters measured after loading; source-reported metrics are not imported.","optional_kernel_packages":{"causal_conv1d":false,"fla":false,"flash_attn":false,"kernels":false},"parameters":511217664,"peak_gpu_bytes":3111938048,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Challenge / Qwen3.5 same-base cohort","catalogue_id":"arc-challenge","choice_score":"sum_log_likelihood","contract":"Question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"qwen35-arc-challenge-full","license":["cc-by-sa-4.0"],"limit":1172,"metric":"accuracy","models":[{"id":"Qwen/Qwen3.5-0.8B-Base","revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68"},{"id":"Qwen/Qwen3.5-0.8B","revision":"2fc06364715b967f1860aea9cf38778875588b17"},{"id":"Mapika/decider-0.8b","revision":"a0a01d6f8135298f400a8c856b355793012ae971"},{"id":"SeerRay-Lab/Xiaomi-OCR-0","revision":"ff990be318c4c68f7e621dd690231200f47e48c4"},{"id":"unsloth/Qwen3.5-0.8B-Base","revision":"44bc982576bf30f0a9440303598859d92604ea5d"},{"id":"nineninesix/qwen3_5-full-attn-only-14","revision":"a26dff91d90e168da8e49f0c19dde98447c095da"},{"id":"OmniJev/PlayJev-0.8B","revision":"a7348002b1e159add7037d6d50812cd4db2d96e9"},{"id":"mithulaartigala/reasoning-summarizer-qwen3.5-0.8b-preview","revision":"558fc651bf01bbe7cf306ebf2fb32032d7b4e818"},{"id":"sh0ck0r/Qwen3.5-0.8B-heretic","revision":"fb35b60374fc15c99a3890e55a5a77e6d015cf4e"},{"id":"McGill-NLP/AfriqueQwen3.5-0.8B-50Langs","revision":"f49099d79e51bbb783fae622877c4c3415142c82"},{"id":"ahmedeltawel/Qwen3.5-0.8B-hsb-dsb","revision":"7e781a3e970474d3fcb215ec0d3c5832e1131b9f"},{"id":"Kewk/Heretical-Qwen3.5-0.8B","revision":"c16807de35d1a569fecbdb240eaaa0ea332372fb"},{"id":"cooperdk/BooruNL-0.8B","revision":"14dbcf2e009f77697dcc7f23e2a706b71b52a5d1"},{"id":"mghafiri/qwen3.5-0.8B-decision-model","revision":"4a9939034a27006b4b62ad5aed65a90b82d0d350"},{"id":"NamanAgnih0tri/AlphaRoute-0.8B-v1.5","revision":"ef87d71f1105f47a15d0d290ee000bcfc0ed68c0"}],"name":"Qwen3.5 0.8B cohort / full ARC-Challenge","parent_task":"sub1b-arc-challenge","priority_models":["Qwen/Qwen3.5-0.8B-Base"],"protocol_version":"codesota-qwen35-arc-challenge-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Challenge/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_qwen35_challenge.py","selection":"All 1172 test items; raw summed conditional answer likelihood; no chat or images; same frozen 14 candidates plus original base control","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":19.207840957911685,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","scope":"8877f1f73ee0a3c138937e07b7f6ca917f2c1d914d223e6a8ffa42b9ac8f0a47","score":0.21416382252559726,"scoring_seconds":14.256044136011042,"seconds":17.62481356598437,"seed":42,"selection":"Full pinned ARC-Challenge test split; all 1172 source-order items","status":"completed","task":"qwen35-arc-challenge-full","text_parameters":511217664,"tf32":false,"truncated_candidates":0,"vision_parameters":0,"worker":"white-gpu0"}