{"acc_norm":0.5909090909090909,"acc_norm_definition":"argmax summed conditional log likelihood / Python character length of answer text, excluding leading delimiter","batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":176,"seconds":0.30506313091609627},{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.13454019895289093},{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.13408031198196113},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.13404732092749327},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.1365584919694811},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.13249841204378754},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.13249587698373944},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.13572379597462714},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.13863649603445083},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.13716956099960953},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.13711338699795306},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.13651506195310503},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.13811913097742945},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.1357606650562957},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.1370390180964023},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.1373501680791378},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.13850318791810423},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.13719801499973983},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13822574098594487},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.1370519819902256},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13573266298044473},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13335995795205235},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13874693296384066},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13575715804472566},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13683402608148754},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13873783592134714},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13436352007556707},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13766109803691506},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.1359837909694761},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13329979707486928},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13243150792550296},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13292609003838152},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13275763590354472},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.1322316851001233},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.1326074959943071},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13148280803579837},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13200352899730206},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13386207493022084},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13254047103691846},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13577408995479345},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.1349796790163964},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.132115107960999},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13243664696346968},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13074868998955935},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13204976601991802},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13296117493882775},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13327948795631528},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.1341060089180246},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.1320980649907142},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13234193995594978},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13312552100978792},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13386714400257915},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.1370840120362118},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13996845996007323},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.1374469290021807},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13640032801777124},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13789381203241646},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13504073198419064},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13315676001366228},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13381548796314746},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13355186593253165},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.1327712000347674},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13318130595143884},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13300467596855015},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.1332586200442165},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13728725200053304},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13475678290706128},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13473477202933282},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13387210306245834},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13360775006003678},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13511394802480936},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13627874106168747},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13861847307998687},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13376141688786447},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1335003210697323},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13361661694943905},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13406076503451914},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1387128490023315},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13787801307626069},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1383615939412266},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13552389305550605},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13577749603427947},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13468769402243197},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1345079990569502},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13401442801114172},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13472304004244506},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13575724803376943},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13816184096504003},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1390607989160344},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13462553801946342},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13606798695400357},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13439695199485868},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13403682003263384},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1337688110070303},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1340721269370988},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1345164350932464},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13715435308404267},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13587671099230647},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13698057003784925},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13926360593177378},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1370357619598508},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13623783504590392},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13656706793699414},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13947731407824904},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.14058989903423935},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13840961398091167},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13521369500085711},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1362805439857766},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13521046901587397},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1385175350587815},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13709587405901402},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13709723704960197},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1370522630168125},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13734743394888937},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13703214598353952},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13749506906606257},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13536232197657228},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13594922609627247},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13549055100884289},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1410693230573088},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13521775300614536},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13433816202450544},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13401932700071484},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13575914199464023},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13552134798374027},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13556410802993923},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13646140100900084},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13487605599220842},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13484998710919172},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13787312398198992},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.14176921790931374},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1370235400972888},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13592220598366112},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13542270509060472},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13436134601943195},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13948218408040702},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13527719303965569},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13441244093701243},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13597182801458985},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13468128198292106},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13625148008577526},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13819741795305163},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1356024199631065},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13670803105924278},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1352691879728809},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13437447999604046},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1350507610477507},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1351691420422867},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13452939793933183},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13376291003078222},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1358868699753657},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1373429950326681},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13821523101069033},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13541871798224747},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13615115301217884},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1392911480506882},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13786367501597852},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.135770594002679},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13466722599696368},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13484407600481063},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13581776106730103},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1357724569970742},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13510583306197077},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13686018495354801},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.135276232031174},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1354138379683718},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13658436003606766},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13910916808526963},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13564914697781205},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13469477696344256},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13506752194371074},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13381441601086408},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13489762507379055},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1352794679114595},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13860143104102463},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13581818202510476},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13509625603910536},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13505738298408687},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13601329503580928},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13492962496820837},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13708619598764926},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1370558600174263},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1398776409914717},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1370505300583318},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13648066692985594},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1360959999728948},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13537859194912016},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.136679808027111},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1352606329601258},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1353005369892344},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13623711396940053},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13625263201538473},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13432274293154478},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13579990796279162},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.135838589980267},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13711549097206444},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13692447496578097},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13554422103334218},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13924316898919642},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13922201900277287},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13484862504992634},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13565039006061852},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13699011597782373},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1344388599973172},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13471614697482437},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13452611304819584},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.14425652497448027},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13815220398828387},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13664591603446752},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13473614398390055},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13536632899194956},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1359830800211057},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13541377801448107},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13616247405298054},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13919883500784636},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1380303771002218},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13916333904489875},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1362075270153582},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1365953308995813},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13670500495936722},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13515687896870077},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13700585602782667},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13534754398278892},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13454590004403144},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1352881940547377},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13631308497861028},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13514458609279245},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13942319410853088},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13559806195553392},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1358917390462011},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.14040159794967622},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13868863403331488},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13788002589717507},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13596759107895195},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13697448803577572},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13699284195899963},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1372383099514991},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13761161593720317},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13566383405122906},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13727729197125882},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1365395070752129},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13689159392379224},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13746922998689115},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13719879603013396},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13537090795580298},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13751508598215878},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13878004392609},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1381443099817261},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13602648000232875},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13999330694787204},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1416909620165825},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14099041593726724},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14128652797080576},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14247677696403116},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1427210329566151},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14577210193965584},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14398923702538013},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1417745070066303},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14140861597843468},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1408070740289986},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1410779890138656},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1416367309866473},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1375880519626662},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13565683097112924},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13642742799129337},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13745416200254112},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13313540001399815},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13601772405672818},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13500579702667892},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13495199696626514},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1336802060250193},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13732928992249072},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13530743005685508},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1383838360197842},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13532767700962722},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13530446391087025},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1360796489752829},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13889621093403548},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13586686295457184},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1358182019321248},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13857227703556418},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13743774103932083},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.135245765093714},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13978338590823114},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1355056599713862},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13535975699778646},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1353525340091437},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13635559403337538},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13518535194452852},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1353739140322432},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13624960591550916},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13543466699775308},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13767475401982665},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1378078720299527},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13716037501581013},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13721107901073992},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13447697996161878},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13620992295909673},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13481886906083673},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1354550850810483},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13728520704898983},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1351926459465176},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1352228719042614},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1369027440669015},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13603817205876112},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1373045629588887},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.137874515959993},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13826212903950363},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.14020507200621068},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1371241370216012},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13646451698150486},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13506906502880156},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13523851102218032},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13688981998711824},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13538858108222485},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13884081807918847},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1350257049780339},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13538393296767026},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13607101293746382},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13525139505509287},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13542075105942786},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13679386104922742},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13620934099890292},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1358417560113594},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1401845730142668},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.14326243300456554},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1356498480308801},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13568624597974122},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13755445997230709},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1356599669670686},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13570161501411349},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1385071959812194},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13789531402289867},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13666493992786855},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13557640102226287},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.14096639095805585},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13700322096701711},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13635678600985557},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13718676404096186},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13888431899249554},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.14035498106386513},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13680179603397846},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13741033105179667},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13743523694574833},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1371248080395162},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13820018398109823},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1358807380311191},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1370844430057332},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1363768340088427},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13753181707579643},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13679776794742793},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13727134198416024},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1375649080146104},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13681852794252336},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13726955803576857},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1394743489800021},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.14199348608963192},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13863176794257015},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13665291899815202},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13617734098806977},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1352976409252733},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13488994201179594},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13623169192578644},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13504539092537016},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13552314194384962},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1388909409288317},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13583063601981848},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13654000801034272},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13522902398835868},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13715170789510012},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13623913703486323},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.14113748003728688},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13735508802346885},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13685027696192265},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13669053802732378},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13839754206128418},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13574184000026435},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.1368558070389554},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.1366934539983049},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13707693899050355},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13608787499833852},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1359116369858384},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13888034201227129},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13989960204344243},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13755622308235615},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1375297229969874},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1356986491009593},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.14028610300738364},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1376585840480402},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13561517396010458},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13690703199245036},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13628629501909018},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13541855697985739},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.1361872999696061},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.135756426025182},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13851331698242575},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.1365625699982047},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.137072611018084},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13554529298562557},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.1383830150589347},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13605161698069423},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13580559799447656},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13860905601177365},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.1392459429334849},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13610206102021039},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13930146698839962},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.1361944630043581},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13802609895356},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.1400952769909054},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.1380720039596781},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13669952494092286},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13614881795365363},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13733439799398184},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13626589707564563},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.1437699580565095},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.1363693099701777},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.1363335329806432},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13693599600810558},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13829591195099056},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13804876094218343},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13499819301068783},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13622706499882042},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13605177693534642},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13623146200552583},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.14208938495721668},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13677208102308214},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13525304791983217},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13668350607622415},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.1365573100047186},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13699522602837533},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.135724046966061},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13600932899862528},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.1370953929144889},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13817200099583715},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.14484527299646288},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.1395799060119316},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.1362741719931364},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13780127000063658},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13607775594573468},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13614896906074136},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.1366107000503689},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13605671597179025},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.1365926560247317},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13678275095298886},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13876403495669365},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13608656299766153},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1366806400474161},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.135838589980267},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13595684000756592},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13723209802992642},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13850473100319505},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13633957400452346},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13569484208710492},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13761014305055141},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1391101200133562},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1361483879154548},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13572549901437014},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13603176001925021},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13870141794905066},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13878964202012867},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13841943198349327},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13561733707319945},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13443730701692402},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13565327401738614},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.14033687696792185},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13744439405854791},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.1385144890518859},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13526090304367244},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13552479504141957},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13583367108367383},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13532426103483886},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13561096601188183},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13630149292293936},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13630932802334428},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13529454602394253},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13869811198674142},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.13534622197039425},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.1359488859307021},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.13525778707116842},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.13613292900845408},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.1374918930232525},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.1390871969051659},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.14351688709575683},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.14330724498722702},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.14862875803373754},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.14584404602646828},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.14449847699142992},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.14563123998232186},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.1443702580872923},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.14102373796049505},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.14225780894048512},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.1410389170050621},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.1387339289067313},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.1374464789405465},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.13860604993533343},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1422437729779631},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.14446815999690443},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.14357014594133943},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.14409260998945683},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1434737159870565},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.14352154498919845},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.140422296943143},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.1432381069753319},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.14048414200078696},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.13716305897105485},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.14027439104393125},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.1361173699842766},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.13902960997074842},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.14035805698949844},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.14052748202811927},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.14001778198871762},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.13782228901982307},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.14060028898529708},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.14199469808954746},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.13712167192716151},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.13692922296468168},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.13872596400324255},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.13804876105859876},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13690931699238718},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13787965499795973},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13621161493938416},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13683768303599209},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.13659045204985887},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1385953800054267},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1369477689731866},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.13707295095082372},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.1375928510678932},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13846247200854123},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.1385719869285822},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.1366765929851681},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13783556292764843},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.1383492809254676},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.1417866910342127},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13800323603209108},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13970211299601942},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13799346797168255},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13692318298853934},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13666569197084755},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13659207499586046},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13546287990175188},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13683041906915605},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13581368408631533},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13891698000952601},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13860504794865847},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.1386684359749779},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.13785845693200827},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.13755789608694613},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.13739522208925337},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.13741039100568742},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.1376291689230129},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.13679428200703114},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.1417894159676507},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.13723048497922719},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.13665627501904964},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.13851521001197398},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.13659959903452545},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.13751647900789976},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.13692440500017256},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.13972777093295008},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.14368638198357075},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.14300370996352285},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.14021567103918642},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.1379129079869017},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.13742179202381521},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.1379872460383922},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.1384278880432248},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.13825624797027558},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.1373164050746709},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.13777451997157186},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.14120789093431085},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.13780376501381397},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.13852167304139584},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.13864422193728387},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.13825187005568296},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.14059638103935868},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.14072126406244934},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.13780298305209726},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.13719611102715135},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.14191311597824097},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.138824537047185},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.1385031290119514},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.13854641001671553},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.13822131289634854},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.13867455709259957},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.1395271170185879},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.13930687704123557},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.14308674400672317},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.13793846499174833},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.1382383849704638},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.14032953407149762},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.14222627005074173},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.14036466996185482},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.15151824604254216},{"candidates":13,"kind":"choice","padded_tokens":2119,"seconds":0.14511152007617056}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-arc-easy-standard-v2","finished":1792205718,"gpu":0,"id":"0d25cdfcbb8238cbd864f8cb","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","model_revision":"7d11e8fd63764baf00c276f586b2aa776648ed33","n":2376,"notes":"Full ARC-Easy test split (2376 examples); FP32, 1024-token input ceiling, raw summed answer-token log likelihood, zero-shot Question/Answer prompt. Character-normalized accuracy reported separately as acc_norm. Native masked-padding adapter uses unpadded row forwards. Distinct from the earlier token-normalized pilot.","peak_gpu_bytes":841644544,"profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Easy (full test split, raw accuracy)","catalogue_id":"arc-easy","choice_score":"sum_log_likelihood","contract":"Question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"sub1b-arc-easy-full","license":["cc-by-sa-4.0"],"limit":2376,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / full sub-1B standard evaluation","parent_task":"reading","protocol_version":"codesota-sub1b-arc-easy-standard-v2","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_standard.py","selection":"Full pinned test split: 2376 examples; raw summed conditional answer likelihood, zero-shot Question/Answer prompt; separate character-normalized diagnostic","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":86.19639519799966,"measurement_version":"gpu-slot-v1"},"runner_sha256":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","scope":"cf50d4b44de811db4aed984c74f66a1eab461af452494ae6c905f9c80d3ffc25","score":0.6216329966329966,"scoring":"argmax raw summed conditional answer-token log likelihood; zero-shot Question/Answer prompt; no chat template or added BOS","seconds":83.2209709190065,"seed":42,"selection":"full pinned ARC-Easy test split; all 2376 examples in source order","status":"completed","task":"sub1b-arc-easy-full","worker":"white-gpu0"}