{"acc_norm":0.2645051194539249,"acc_norm_definition":"argmax summed conditional likelihood / Python character length of answer, excluding leading delimiter","acc_raw":0.22610921501706485,"batch_policy":{"answer_limit":512,"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.3160991739714518},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.1183766929898411},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.12082219298463315},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.1151944799348712},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.11829204496461898},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.11891115992330015},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.11650618503335863},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.11595323402434587},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.11649652803316712},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.11453910695854574},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.11518011300358921},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.11710365896578878},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.11686166899744421},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.11466941004619002},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.11642033501993865},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.11575254006311297},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.11696286697406322},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.11737373296637088},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.11486819095443934},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.11615003109909594},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11483589094132185},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11606442101765424},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11566523707006127},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11710800800938159},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11797268991358578},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11639370501507074},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11661204195115715},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11775296996347606},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1178968989988789},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11770357901696116},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11605666694231331},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11542368703521788},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11642133700661361},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.12305590906180441},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11518975999206305},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11573988606687635},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11606127594131976},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11675966798793525},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11746509408112615},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11529512703418732},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11558904498815536},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1157072460046038},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11830618197564036},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11699761101044714},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11668022896628827},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1171209920430556},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11723309196531773},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11727544001769274},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11730503500439227},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1173525940394029},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11499730101786554},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11898207303602248},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11999459995422512},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11550194292794913},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11567412305157632},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11756228504236788},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11690737400203943},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1163073240313679},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11786389700137079},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11821042303927243},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12002377503085881},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12018605798948556},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11577005300205201},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11749479896388948},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11752640793565661},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11762350902426988},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11777007207274437},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.12050054501742125},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.12329524604137987},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11650417104829103},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11792284704279155},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11832121992483735},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11708283994812518},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.12199095403775573},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1174990669824183},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11701684794388711},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11720095097552985},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11508427397347987},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11681282694917172},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11806106509175152},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11646934598684311},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11628762795589864},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11646682198625058},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11676629108842462},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11714757198933512},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11848896299488842},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11542469798587263},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11682386801112443},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11788285290822387},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11721950501669198},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11650048405863345},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11717757803853601},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11713206197600812},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11684136092662811},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1169851979939267},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11514033796265721},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11569989193230867},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11774115904700011},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11670120910275728},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11620194790884852},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11603623896371573},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11755386891309172},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11509346100501716},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11931072501465678},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11995169101282954},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11811823199968785},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.12012706801760942},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11862059799022973},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11687204800546169},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11898461799137294},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11672550393268466},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11726393806748092},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11726795602589846},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11704711499623954},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11772584996651858},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11814771697390825},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1197662950726226},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11696686397772282},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11800881696399301},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11663791094906628},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.12094735691789538},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.12120580906048417},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11727433803025633},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1188449669862166},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11717390094418079},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11911784601397812},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11982738797087222},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.12110642401967198},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11971433798316866},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11813399102538824},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.12149703304748982},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11989405297208577},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.12080497201532125},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.12336689897347242},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.12217727105598897},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.12075862509664148},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.11906462605111301},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.1195090850815177},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.11721568799111992},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.11764093197416514},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.12053824495524168},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.11847567791119218},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1178479470545426},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1183633980108425},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.11879625602159649},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.12686914997175336},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1252076820237562},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11922331200912595},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.12471958296373487},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.12152935296762735},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11743324401322752},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11882595100905746},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.12170623301062733},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11952409404329956},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.12204992398619652},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.1202793619595468},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.12106861406937242},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.11921336397062987},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.11939511192031205},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.12044471001718193},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.12078664707951248},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.11883464700076729},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.12032412504777312},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.1196568210143596},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.11901022505480796},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.12024884496349841},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.11988355300854892},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.12081183504778892},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.11895362904760987},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.12052645196672529},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.12113313400186598},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11972680094186217},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.12144116801209748},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.12061554903630167},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.12177218601573259},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.12014377804007381},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.12051576201338321},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.12042486295104027},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.12151965498924255},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.12279161694459617},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.12125840701628476},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.11838783393613994},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1203848080476746},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.11902075400575995},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.12024366506375372},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.12028729706071317},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.1186953280121088},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.12224370497278869},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11881645407993346},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11783867003396153},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11731906095519662},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11755887907929718},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11923531501088291},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11884888401255012},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11846462800167501},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11828549299389124},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11807595996651798},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11838281201198697},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11956230306532234},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11926176096312702},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11913193296641111},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.12157513597048819},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.12030144198797643},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.12142611993476748},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.12368764798156917},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.1245971629396081},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.12356150208506733},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11833063699305058},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11819806101266295},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.1204255340853706},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.11905028996989131},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.118049904005602},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.11782385199330747},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.11787392606493086},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.1175238429568708},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.11748836701735854},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.1175139540573582},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.11744313302915543},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.1183965610107407},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.11994500807486475},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.11891199101228267},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.11985830601770431},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.1207428959896788},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12084724102169275},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12082761398050934},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.1190088929142803},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.11902747699059546},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.12035773799289018},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.1194875460350886},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.12219108699355274},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.12378279492259026},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.12069019791670144},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11881337803788483},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1188036899548024},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11949230392929167},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.12092532694805413},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11971025995444506},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11929119902197272},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.12104285496752709},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11911189509555697},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11854692094493657},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.12115401297342032},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.1196069079451263},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11922763998154551},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11999131389893591},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.11975437309592962},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.11900577589403838},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.12022912804968655},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.11938144697342068},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.11932938091922551},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.1209852279862389},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.1194974429672584},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.11920990806538612},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.12064543401356786},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.12024467694573104},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.12219809903763235},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.11925780703313649},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.12062323209829628},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.12280438106972724},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.1267800850328058},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.12535876501351595},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.12442800891585648},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.12383939092978835},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.12409287306945771},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.12490697100292891},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.12423706206027418},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.12501368101220578},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.12552906200289726},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.12412677693646401},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.12638811394572258},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.12715786893386394},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.12912610895000398},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.12707607599440962},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.12800560996402055},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.12745877995621413},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.12886094499845058},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.1281290400074795},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.12951520492788404},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.1281256729271263},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.1291332229739055},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.12817211099900305},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.12835186498705298},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.12579972599633038},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.12596976407803595},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.12551279098261148},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.125557434046641},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.12630682194139808},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.12878350995015353},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.12889237294439226},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.12931660399772227},{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.12652694201096892},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.12732215598225594},{"candidates":15,"kind":"choice","padded_tokens":2595,"seconds":0.12998950900509953}],"chat_template":false,"data_loading_seconds":0.04689081397373229,"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"common-core-profiles.json":"a54c99201b8e0784bc5d7efab18b39f3dc50168ff6f0f7bee1315e77bb9bb4f1","evaluate_qwen35_challenge.py":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa","sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"codesota-common-core-native-arc-challenge-v1","finished":1792249416,"gpu":0,"id":"a2d1c9f799c2c87975a9a973","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer139-1.01b","model_load_seconds":1.9401281969621778,"model_revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633","n":1172,"notes":"Full pinned split; primary raw summed conditional answer-token log likelihood accuracy; character-normalized accuracy is a separate diagnostic. Explicit pinned prompt schema, no chat/BOS. Left context truncated only if prompt plus answer exceeds 1024 tokens. Generic text diagnostic does not validate specialist declared tasks.","parameters":139279760,"peak_gpu_bytes":856281088,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Challenge (full test, raw likelihood accuracy)","catalogue_id":"arc-challenge","choice_score":"sum_log_likelihood","contract":"Provided context/question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"native-arc-challenge-common-core-full","license":["cc-by-sa-4.0"],"limit":1172,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Challenge / full sub-1B common core (native)","parent_task":"reading","protocol_version":"codesota-common-core-native-arc-challenge-v1","recipe":{"data_file":"ARC-Challenge/test-00000-of-00001.parquet","schema":"arc-challenge-v1"},"reference_protocol":{"repository":"EleutherAI/lm-evaluation-harness","revision":"d6de81643928d653435c431bae19945d41d32520","task":"arc_challenge"},"runner_file":"evaluate_common_core.py","runtime_backend":"native","selection":"All 1172 pinned test rows in source order; raw answer log likelihood; no chat; character-normalized diagnostic separately","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":39.23847216600552,"measurement_version":"gpu-slot-v1"},"runner_sha256":"8ba81403e69d93b8121bd05e0ea4f9c55e0dc5c721449f50adfda136d7d976f1","scope":"e7984b7ea3a9cae0996b0a453de457a6a7d717e9a2c3e9a95d80f697f8bd5afb","score":0.22610921501706485,"scoring_seconds":35.5111502900254,"seconds":37.60478996101301,"seed":42,"selection":"All 1172 pinned test rows in source order; raw answer log likelihood; no chat; character-normalized diagnostic separately","status":"completed","task":"native-arc-challenge-common-core-full","tf32":false,"truncated_candidates":0,"worker":"white-gpu0"}