{"acc_norm":0.30119453924914674,"acc_norm_definition":"argmax summed conditional likelihood / Python character length of answer, excluding leading delimiter","acc_raw":0.26621160409556316,"batch_policy":{"answer_limit":512,"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.303300156025216},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.12000566103961319},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.12098365498241037},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.12011641694698483},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.1197979940334335},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.12016477796714753},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.118270153994672},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.11810435494408011},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.11837580206338316},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.11771844595205039},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.12050635507330298},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.12425277102738619},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.12089657306205481},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.12126122298650444},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.11811720894183964},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.11942092096433043},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1195266880095005},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.12017465592361987},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1205234460067004},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.12041909201070666},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1176158549496904},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11776752700097859},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1164868000196293},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11733062204439193},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11703872901853174},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11683756404090673},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11565841501578689},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11650499200914055},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11613415204919875},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11632247199304402},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11741505993995816},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11960712797008455},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1205013949656859},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11774248094297945},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11626003601122648},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11756308691110462},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11820274905767292},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11658584396354854},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11643600498791784},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11712026002351195},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1180952179711312},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1174865229986608},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11701580602675676},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1172074630158022},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11748812696896493},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1174524090019986},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11733448901213706},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11762277805246413},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11688051302917302},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11852812592405826},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.12219265999738127},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11758868407923728},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.12066989997401834},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.12054696097038686},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12090502795763314},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11718259600456804},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11681163508910686},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1161188919795677},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11633962404448539},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11810185096692294},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11661585990805179},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12160535505972803},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1160917419474572},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11716018500737846},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11658069398254156},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11635965202003717},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11680894088931382},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11702191608492285},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11813554400578141},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11580252298153937},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1177810929948464},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11504577193409204},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11537001701071858},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11648716998752207},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11449270998127759},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11515740002505481},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11601156299002469},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11444630404002964},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11628737696446478},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11582762992475182},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11640021798666567},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11744891304988414},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11903816706035286},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1174721569987014},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11487569496966898},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11625872401054949},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11777473101392388},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11709931201767176},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1182702750666067},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11766781203914434},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11797255999408662},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11873564298730344},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11685855302494019},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11780137091409415},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11731341108679771},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11656929308082908},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11555258696898818},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11657627602107823},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11477115005254745},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11559818196110427},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.114758066018112},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11511312797665596},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11556938791181892},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11654144094791263},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11701650696340948},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1170784329297021},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11536622897256166},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11674442002549767},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11853092105593532},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11814052297268063},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11805740802083164},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11806701601017267},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11795301304664463},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11775649699848145},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11786411807406694},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11735681199934334},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11732155596837401},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1191969329956919},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11527484899852425},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1165388859808445},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11516266001854092},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11683326598722488},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11876462597865611},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.12065089400857687},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11776294896844774},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11629959102720022},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11573143000714481},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11669987591449171},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11601668293587863},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11565372499171644},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11633843299932778},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11765406606718898},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11801054095849395},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.11852398794144392},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.12222226394806057},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.11946944100782275},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.1175090050091967},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.11883524898439646},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.11819517507683486},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.11721721105277538},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.11592600308358669},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.11796080810017884},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.11812251899391413},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1176506889751181},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.11728580901399255},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.11594719300046563},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.11690828506834805},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11676140199415386},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11471050593536347},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11633611703291535},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11670182005036622},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11759858205914497},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11412764003034681},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11467152391560376},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.1160781669896096},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.11655451508704573},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.11665111593902111},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.11689040204510093},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.1166876939823851},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.11593976896256208},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.11645369697362185},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.1157586919143796},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.11601581098511815},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.1167635649908334},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.11485091794747859},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.11576030403375626},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.11686816101428121},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.11469345493242145},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.11584865895565599},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.1164964169729501},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.11614880908746272},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11575491400435567},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11553336190991104},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.1156850130064413},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11708033597096801},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11634405294898897},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11474424006883055},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11528138199355453},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.11815567093435675},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.11749560001771897},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.11655687994789332},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.11759717005770653},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.11441249097697437},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.11629412998445332},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.11648837092798203},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.11651010299101472},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.11678107804618776},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.11588174104690552},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11576665693428367},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11597087699919939},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11664560495410115},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11680300906300545},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11697230499703437},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11469659104477614},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11574146908242255},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11614107503555715},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11886331101413816},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.12145409197546542},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11621123598888516},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11806986096780747},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11743347498122603},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11704711394850165},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.1179159750463441},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11789957399014384},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11728786304593086},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11822281708009541},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11857941199559718},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11741127306595445},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11893412296194583},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.1204953440465033},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.11825913400389254},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.11651273793540895},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.11680203699506819},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.11646139097865671},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.11720170197077096},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.11937953403685242},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12052238499745727},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.11582157807424664},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.11920842505060136},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.1451302949571982},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12146692594978958},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12209449696820229},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.11719900707248598},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.11865186609793454},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.11687448190059513},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.11657099600415677},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.11851601290982217},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.115642064018175},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.1162337779533118},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.11629508202895522},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.11555466102436185},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11569737701211125},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1162842110497877},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11841225007083267},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1167869089404121},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11694770900066942},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11713137105107307},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11814610403962433},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11986582993995398},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11789244099054486},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11576830910053104},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11627666710410267},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11643719708081335},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11770718498155475},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11643268808256835},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.11660725402180105},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.11689470894634724},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.11662236193660647},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.11735779407899827},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.11592738598119467},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.11598513298667967},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.11795018904376775},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.11723546602297574},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.11680288799107075},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.1181777719175443},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.11920949700288475},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.11586441798135638},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.11736173101235181},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.11704621196258813},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.11667971906717867},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.1169101680861786},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.11810552806127816},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.11680196702945977},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.11547164595685899},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.11602340394165367},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.11676410702057183},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.11707453499548137},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.11863086698576808},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.11861049907747656},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.12299471604637802},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.12294572393875569},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.12036280706524849},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.12131108599714935},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.1211530810687691},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.1215424370020628},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.1220552830491215},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.12185448908712715},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.12330939201638103},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.12430346594192088},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.12258573295548558},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.12335805303882807},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.12462323193904012},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.12416825396940112},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.12430518900509924},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.12482155195903033},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.12520531797781587},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.12439120991621166},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.1267150129424408},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.12611826008651406},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.1259676100453362},{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.12420916999690235},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.12366210995242},{"candidates":15,"kind":"choice","padded_tokens":2595,"seconds":0.12946359906345606}],"chat_template":false,"data_loading_seconds":0.04729607992339879,"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"common-core-profiles.json":"a54c99201b8e0784bc5d7efab18b39f3dc50168ff6f0f7bee1315e77bb9bb4f1","evaluate_qwen35_challenge.py":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa","sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"codesota-common-core-native-arc-challenge-v1","finished":1792249855,"gpu":0,"id":"fd572ea4bbb7231f35cd5ed8","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer139-1.01","model_load_seconds":1.8897882809396833,"model_revision":"2e72c696d7c5cad73dc58606b0751b911dc94495","n":1172,"notes":"Full pinned split; primary raw summed conditional answer-token log likelihood accuracy; character-normalized accuracy is a separate diagnostic. Explicit pinned prompt schema, no chat/BOS. Left context truncated only if prompt plus answer exceeds 1024 tokens. Generic text diagnostic does not validate specialist declared tasks.","parameters":139279760,"peak_gpu_bytes":856281088,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Challenge (full test, raw likelihood accuracy)","catalogue_id":"arc-challenge","choice_score":"sum_log_likelihood","contract":"Provided context/question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"native-arc-challenge-common-core-full","license":["cc-by-sa-4.0"],"limit":1172,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Challenge / full sub-1B common core (native)","parent_task":"reading","protocol_version":"codesota-common-core-native-arc-challenge-v1","recipe":{"data_file":"ARC-Challenge/test-00000-of-00001.parquet","schema":"arc-challenge-v1"},"reference_protocol":{"repository":"EleutherAI/lm-evaluation-harness","revision":"d6de81643928d653435c431bae19945d41d32520","task":"arc_challenge"},"runner_file":"evaluate_common_core.py","runtime_backend":"native","selection":"All 1172 pinned test rows in source order; raw answer log likelihood; no chat; character-normalized diagnostic separately","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":38.771843342110515,"measurement_version":"gpu-slot-v1"},"runner_sha256":"8ba81403e69d93b8121bd05e0ea4f9c55e0dc5c721449f50adfda136d7d976f1","scope":"e7984b7ea3a9cae0996b0a453de457a6a7d717e9a2c3e9a95d80f697f8bd5afb","score":0.26621160409556316,"scoring_seconds":35.00551012600772,"seconds":37.05090898601338,"seed":42,"selection":"All 1172 pinned test rows in source order; raw answer log likelihood; no chat; character-normalized diagnostic separately","status":"completed","task":"native-arc-challenge-common-core-full","tf32":false,"truncated_candidates":0,"worker":"white-gpu0"}