{"acc_norm":0.2986348122866894,"acc_norm_definition":"argmax summed conditional likelihood / Python character length of answer, excluding leading delimiter","acc_raw":0.27474402730375425,"batch_policy":{"answer_limit":512,"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":176,"seconds":0.3122172959847376},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.13691381202079356},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.1355799629818648},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.13712607801426202},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.13632758799940348},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.1372661490458995},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.14125851599965245},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13692896906286478},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13786295196041465},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13546669203788042},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.1341522700386122},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.1358986069681123},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13505302101839334},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13719029794447124},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.1390377830248326},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13838583801407367},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.1397874419344589},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13654328091070056},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.1379891469841823},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1365585889434442},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13780631602276117},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1382870429661125},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.137594681000337},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.14703237696085125},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.14023622893728316},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13881625107023865},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.138018993078731},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.138124689925462},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13776875601615757},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1409702179953456},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.14066797494888306},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13753249496221542},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13732348603662103},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13643719209358096},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13942447502631694},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13775480003096163},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13678268704097718},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13519315200392157},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13681966601870954},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13610471202991903},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13723890797700733},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1356007819995284},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1357603290816769},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13759707601275295},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.14346553396899253},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.14091780106537044},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13747343595605344},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.137489726068452},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13903591001871973},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1388547319220379},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1390538130654022},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13843904598616064},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13606052997056395},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.14043201506137848},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13711203099228442},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1368756500305608},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1372312030289322},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13665870507247746},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13801824196707457},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13712104794103652},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13944552408065647},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13843077095225453},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1410859340103343},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1366202839417383},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1379560650093481},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13839498395100236},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1376983750378713},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13589949894230813},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13743135600816458},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13722992199473083},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13718783401418477},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1370889199897647},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13889607100281864},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1392173379426822},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13970287493430078},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13946754508651793},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13928932207636535},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13800207106396556},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13715871796011925},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13787090696860105},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.138176305918023},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1375340078957379},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1377696980489418},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1367229059105739},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1378083899617195},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1380538180237636},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13880354701541364},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1401216359809041},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13945779693312943},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13989233691245317},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14176818705163896},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1425141179934144},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13858964701648802},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13856089406181127},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1395000860793516},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1377849670825526},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13743288000114262},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13732095097657293},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1418749049771577},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13946586195379496},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13958632701542228},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1388472180115059},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13774660509079695},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13829060993157327},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13894850702490658},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13794657797552645},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13942632789257914},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.14085751806851476},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13824874197598547},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13918296399060637},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13716039201244712},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1369394090725109},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13607375498395413},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1361591840395704},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13725211191922426},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13676624605432153},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.14284219092223793},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13819214596878737},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13839606707915664},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1376293069915846},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13775012094993144},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13678930897731334},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13989194703754038},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1411731969565153},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.14288095210213214},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13921246898826212},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13792186102364212},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1373924439540133},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1377910979790613},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13684210798237473},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1380213670199737},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13668810995295644},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13772588595747948},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.137489645043388},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1415843339636922},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1379263399867341},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13797040202189237},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13769244391005486},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13921380206011236},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13857043196912855},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13711063901428133},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13807875395286828},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.1371411350555718},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.14239415398333222},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.1379042889457196},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13859801297076046},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1371579470578581},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13993693108204752},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1398982290411368},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1398143619298935},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13784181303344667},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1378104139585048},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13966969202738255},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.14038560795597732},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13790871808305383},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13764357299078256},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13645971403457224},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13742776005528867},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13762758299708366},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13857976894360036},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13656233495566994},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.14116449002176523},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.1374746080255136},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.138080958975479},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.1366234899032861},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13823035696987063},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.1376708640018478},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13748635002411902},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.1382452449761331},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13943956303410232},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.1424382069380954},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13674432504922152},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.1374113989295438},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.1401344999903813},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.1397932330146432},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13573375006671995},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.1371786049567163},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.1372070990037173},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13758997293189168},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13701063196640462},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13915890897624195},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13845142000354826},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13701449905056506},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1382203489774838},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.137572480016388},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1386779519962147},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13820000004488975},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.1373476809822023},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.1380447510164231},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.1401024489896372},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13721155701205134},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13772462401539087},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13838347292039543},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13685979100409895},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13849599298555404},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.136847426998429},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13802816008683294},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.13796625402756035},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.14349606994073838},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.1407526929397136},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.1389363140333444},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.1381785209523514},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.13638826191890985},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.13784163305535913},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.13924262509681284},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.1386067890562117},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.13784301397390664},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.1407403500052169},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.13940544903744012},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.1375987189821899},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.1375857749953866},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.13734054693486542},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.13877686695195735},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.1358676600502804},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.13696561800315976},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.13802230905275792},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.14288744505029172},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1372618399327621},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.13727922306861728},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.1370731689967215},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.13788799790199846},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.13745371799450368},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.13853544706944376},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.1423775339499116},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.14143840200267732},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.14062716893386096},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.13821237394586205},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.139853265020065},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13658548891544342},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13761581107974052},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13636652007699013},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.14036576997023076},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.13916001096367836},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.13664842606522143},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.14019883901346475},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1383182720746845},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13691796897910535},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13793165003880858},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.1378512899391353},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13738704402931035},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.1369828499155119},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13899261900223792},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13955116108991206},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.14016286202240735},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13698913203552365},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.1374289320083335},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13862953206989914},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.14185949706006795},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13916686503216624},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13823778089135885},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.1368467570282519},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13784392701927572},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.14008645003195852},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.13783849694300443},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.13733868394047022},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.13915681489743292},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.13970143196638674},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.13726701005361974},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.136344869970344},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.13776714296545833},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.13667865190654993},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.14173427294008434},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.13859450595919043},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.13946465007029474},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.13613223296124488},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.13843892596196383},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.1384797419887036},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.1368535589426756},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.1385974120348692},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.1367574700852856},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.14190354000311345},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.14143316203262657},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.1409726629499346},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.1415225879754871},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.1398089820286259},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.13951537397224456},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.13824463391210884},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.13900884904433042},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.14638317609205842},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.14115524396765977},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.14075779204722494},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.14106507506221533},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.14059563900809735},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.13973580591846257},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.14049646502826363},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.14077657705638558},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.14328839303925633},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.14001484599430114},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.14044102106709033},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.14083171007223427},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.14169100404251367},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.1418922080192715},{"candidates":15,"kind":"choice","padded_tokens":2610,"seconds":0.16104260191787034}],"chat_template":false,"data_loading_seconds":0.04780756100080907,"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"common-core-profiles.json":"a54c99201b8e0784bc5d7efab18b39f3dc50168ff6f0f7bee1315e77bb9bb4f1","evaluate_qwen35_challenge.py":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa","sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"codesota-common-core-native-arc-challenge-v1","finished":1792250741,"gpu":0,"id":"cc339a1fce7a8ceec557557f","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","model_load_seconds":1.8845021929591894,"model_revision":"7d11e8fd63764baf00c276f586b2aa776648ed33","n":1172,"notes":"Full pinned split; primary raw summed conditional answer-token log likelihood accuracy; character-normalized accuracy is a separate diagnostic. Explicit pinned prompt schema, no chat/BOS. Left context truncated only if prompt plus answer exceeds 1024 tokens. Generic text diagnostic does not validate specialist declared tasks.","parameters":149333081,"peak_gpu_bytes":894092288,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Challenge (full test, raw likelihood accuracy)","catalogue_id":"arc-challenge","choice_score":"sum_log_likelihood","contract":"Provided context/question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"native-arc-challenge-common-core-full","license":["cc-by-sa-4.0"],"limit":1172,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Challenge / full sub-1B common core (native)","parent_task":"reading","protocol_version":"codesota-common-core-native-arc-challenge-v1","recipe":{"data_file":"ARC-Challenge/test-00000-of-00001.parquet","schema":"arc-challenge-v1"},"reference_protocol":{"repository":"EleutherAI/lm-evaluation-harness","revision":"d6de81643928d653435c431bae19945d41d32520","task":"arc_challenge"},"runner_file":"evaluate_common_core.py","runtime_backend":"native","selection":"All 1172 pinned test rows in source order; raw answer log likelihood; no chat; character-normalized diagnostic separately","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":44.7007399010472,"measurement_version":"gpu-slot-v1"},"runner_sha256":"8ba81403e69d93b8121bd05e0ea4f9c55e0dc5c721449f50adfda136d7d976f1","scope":"e7984b7ea3a9cae0996b0a453de457a6a7d717e9a2c3e9a95d80f697f8bd5afb","score":0.27474402730375425,"scoring_seconds":41.0111145300325,"seconds":43.053321058047004,"seed":42,"selection":"All 1172 pinned test rows in source order; raw answer log likelihood; no chat; character-normalized diagnostic separately","status":"completed","task":"native-arc-challenge-common-core-full","tf32":false,"truncated_candidates":0,"worker":"white-gpu0"}