{"acc_norm":0.27986348122866894,"acc_norm_definition":"argmax summed conditional likelihood / Python character length of answer, excluding leading delimiter","acc_raw":0.26109215017064846,"batch_policy":{"answer_limit":512,"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":176,"seconds":0.3105818539625034},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.13533567695412785},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.1380919589428231},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.13858880603220314},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.13862454204354435},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13470486109144986},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13431463297456503},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.1347298480104655},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13493004092015326},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13483850995544344},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13491277897264808},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.1338873360073194},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13433841709047556},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.1345715920906514},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.1357966479845345},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13559252605773509},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13669676601421088},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13656111294403672},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13575621193740517},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13674232189077884},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13833289907779545},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13566175603773445},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13518299290444702},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13550252898130566},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13609911198727787},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13774165499489754},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13747720199171454},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13904853304848075},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13967333908658475},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13934731110930443},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13851783296559006},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13884332100860775},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13786538608837873},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13661390193738043},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13642209395766258},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13908917002845556},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1377290430245921},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1372112960088998},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.14066098199691623},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.14132790500298142},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1399989560013637},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13921292999293655},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13724233501125127},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13727511593606323},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13795492297504097},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13937088497914374},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13580122601706535},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1368946760194376},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1376424509799108},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1369478349806741},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1372007360914722},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13982071401551366},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1412162269698456},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1395953840110451},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13754463801160455},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13678235607221723},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13752297696191818},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1375501889269799},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13784058997407556},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13619479991029948},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13830421504098922},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1373174439650029},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13679659308400005},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13873964792583138},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13818174693733454},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.14041430107317865},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13926543795969337},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13918674096930772},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.14085740805603564},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1422759039560333},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.14502599404659122},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.141915310989134},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13918634003493935},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.14053334400523454},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.14185005892068148},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1394248950527981},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13774751592427492},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1373828270006925},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13723679399117827},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13655685598496348},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13689816207624972},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13599546800833195},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13844872498884797},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13637860293965787},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13552297709975392},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13635286502540112},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13731527095660567},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13693648402113467},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13703628000803292},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13698091602418572},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1358294279780239},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13586007594130933},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13627780601382256},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13609513395931572},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13638062693644315},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1352662880672142},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.135719392914325},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13601263996679336},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13666753098368645},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13477297802455723},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13562032801564783},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13646249007433653},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13850810599979013},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.14254634792450815},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1402910710312426},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1375329359434545},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13572187803220004},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13548685901332647},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13749080803245306},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13829074997920543},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13750963297206908},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13744639500509948},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13694413797929883},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13646113697905093},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13665673194918782},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1351675030309707},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13545855705160648},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13516348705161363},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13758156704716384},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13761698303278536},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1369674310553819},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.137120786937885},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13671674393117428},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13668564602266997},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13682244101073593},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13695720210671425},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1365148170152679},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1382246259599924},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13821426697541028},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1389806669903919},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13621583895292133},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1361992279998958},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13739227398764342},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1373453459236771},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13734171900432557},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13667643896769732},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13706404191907495},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1369433259824291},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13678964995779097},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.14001063804607838},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.14008971501607448},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13967049401253462},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13766296894755214},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.136695254012011},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.1368053489131853},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13705937296617776},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13531992794014513},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.136359907919541},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13500274694524705},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13568542001303285},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13493241497781128},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13504825090058148},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.136449234909378},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.137023946037516},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13755893509369344},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13594133802689612},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13598902605008334},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13578508596401662},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13508202496450394},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13509385706856847},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13575569097884},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13871165609452873},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13658990699332207},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13701763504650444},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13638957403600216},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.1355325150070712},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13591158110648394},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.135549655999057},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13541538605932146},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13456615200266242},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13478321698494256},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13484294805675745},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13550221803598106},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13731458806432784},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13911096996162087},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.136288725072518},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13749998505227268},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13982242706697434},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.1397767219459638},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.1366238099290058},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13540270295925438},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13528572500217706},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13483874092344195},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13449774496257305},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13482866098638624},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13487639103550464},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13490756892133504},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13455354899633676},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13469203701242805},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13486780493985862},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.1355054440209642},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13654190697707236},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.1374253659741953},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.1389397500315681},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.1390912930946797},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13779368298128247},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13663908909074962},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.13656355801504105},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.1374790060799569},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.13667557702865452},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.13741236098576337},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.13920034701004624},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.13978169101756066},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.14055830996949226},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.13796359999105334},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.13867561798542738},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.13754044997040182},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.13844005903229117},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.14222534908913076},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.13671200501266867},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.13588176597841084},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.13930180598981678},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.14929005899466574},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.14327496709302068},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.13634808594360948},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1533762210747227},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1521264100447297},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.13840001402422786},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.1383483080426231},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.13795884093269706},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.1367807239294052},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.13725581904873252},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.13861781009472907},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.14007598999887705},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.1382936049485579},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.1365463660331443},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.1390637409640476},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.13893936993554235},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13896276394370943},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13754835503641516},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13755057891830802},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13724769395776093},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.13759012310765684},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1375575619749725},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.13735303096473217},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1368302860064432},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13760723499581218},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13840793899726123},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13801553705707192},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.1385522880591452},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13777048897463828},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.1373115130700171},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13740030897315592},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.1370984959648922},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13766047405079007},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.1371368080144748},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.1375621409388259},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13736068608704954},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13778392493259162},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13756074896082282},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13714318990241736},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.1430598160950467},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.14208907494321465},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.13999374699778855},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.13931135402526706},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.1385812220396474},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.1379098700126633},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.1378301199292764},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.13721144699957222},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.1371712009422481},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.13733096001669765},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.13615314301569015},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.1368177420226857},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.13609722792170942},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.13646327098831534},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.13618929008953273},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.13673098105937243},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.13680387602653354},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.13680514798033983},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.13661868101917207},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.1401024399092421},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.13786529598291963},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.13779352197889239},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.13780387106817216},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.13757672801148146},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.13711771205998957},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.1374780839541927},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.13927489600609988},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.13993202103301883},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.1387610170058906},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.13891514495480806},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.13849408901296556},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.1379276029765606},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.13893901905976236},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.13765501393936574},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.13921340100932866},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.14039304200559855},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.13959272799547762},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.14012335799634457},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.1396796709159389},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.13936495408415794},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.13936506398022175},{"candidates":15,"kind":"choice","padded_tokens":2610,"seconds":0.1462822479661554}],"chat_template":false,"data_loading_seconds":0.07095442898571491,"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"common-core-profiles.json":"a54c99201b8e0784bc5d7efab18b39f3dc50168ff6f0f7bee1315e77bb9bb4f1","evaluate_qwen35_challenge.py":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa","sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"codesota-common-core-native-arc-challenge-v1","finished":1792250273,"gpu":1,"id":"3f2816bfe776578a9ba35efc","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer-149M-Decision-v1","model_load_seconds":1.9777436279691756,"model_revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7","n":1172,"notes":"Full pinned split; primary raw summed conditional answer-token log likelihood accuracy; character-normalized accuracy is a separate diagnostic. Explicit pinned prompt schema, no chat/BOS. Left context truncated only if prompt plus answer exceeds 1024 tokens. Generic text diagnostic does not validate specialist declared tasks.","parameters":149333081,"peak_gpu_bytes":894092288,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Challenge (full test, raw likelihood accuracy)","catalogue_id":"arc-challenge","choice_score":"sum_log_likelihood","contract":"Provided context/question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"native-arc-challenge-common-core-full","license":["cc-by-sa-4.0"],"limit":1172,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Challenge / full sub-1B common core (native)","parent_task":"reading","protocol_version":"codesota-common-core-native-arc-challenge-v1","recipe":{"data_file":"ARC-Challenge/test-00000-of-00001.parquet","schema":"arc-challenge-v1"},"reference_protocol":{"repository":"EleutherAI/lm-evaluation-harness","revision":"d6de81643928d653435c431bae19945d41d32520","task":"arc_challenge"},"runner_file":"evaluate_common_core.py","runtime_backend":"native","selection":"All 1172 pinned test rows in source order; raw answer log likelihood; no chat; character-normalized diagnostic separately","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":44.54480325104669,"measurement_version":"gpu-slot-v1"},"runner_sha256":"8ba81403e69d93b8121bd05e0ea4f9c55e0dc5c721449f50adfda136d7d976f1","scope":"e7984b7ea3a9cae0996b0a453de457a6a7d717e9a2c3e9a95d80f697f8bd5afb","score":0.26109215017064846,"scoring_seconds":40.73565659194719,"seconds":42.89309810008854,"seed":42,"selection":"All 1172 pinned test rows in source order; raw answer log likelihood; no chat; character-normalized diagnostic separately","status":"completed","task":"native-arc-challenge-common-core-full","tf32":false,"truncated_candidates":0,"worker":"white-gpu1"}