{"acc_norm":0.5433501683501684,"acc_norm_definition":"argmax summed conditional log likelihood / Python character length of answer text, excluding leading delimiter","batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":176,"seconds":0.3087137369439006},{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.13470637891441584},{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.13577545201405883},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.13832068804185838},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.13667087198700756},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.13482317700982094},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.1352985929697752},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.13612353103235364},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.13413720799144357},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.13939139409922063},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.13757399492897093},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.13549229397904128},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.13442771905101836},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.13333531201351434},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.1332308780401945},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.13607860705815256},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.13593247497919947},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.13522083801217377},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13706091896165162},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.135205319034867},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13566313299816102},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13651719503104687},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13607614301145077},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13697562995366752},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.1357522790785879},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13983002200257033},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13811865204479545},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.1358257959363982},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.1389846559613943},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.1381543279858306},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.138959298026748},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13871290907263756},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13564502901863307},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.1357880359282717},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.137582600931637},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13627305999398232},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13713678088970482},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13521974603645504},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13522931397892535},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.1371083080302924},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13516937207896262},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13974661706015468},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13449037598911673},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.1337180349510163},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13633226102683693},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13442085694987327},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.1340089678997174},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.1347241719486192},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13419733999762684},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13560862094163895},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13471071689855307},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13639290304854512},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.1369944349862635},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.14042355900164694},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13867040001787245},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13897980703040957},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.1400833639781922},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.14002093893941492},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13627082598395646},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.1371559159597382},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.1356919870013371},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.1361611420288682},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13808869500644505},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13691629003733397},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.136065804050304},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13677480502519757},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13570008194074035},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13610012794379145},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13719116093125194},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13602364505641162},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13670023693703115},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13960229803342372},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1419578300556168},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13877299102023244},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13715197902638465},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13707405398599803},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13706000801175833},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13749288499820977},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1382751630153507},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1391130150295794},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13936719996854663},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.14147903700359166},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1368879369692877},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13871588499750942},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1367023119237274},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13904562103562057},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13758500793483108},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13658075605053455},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1401367160724476},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13810699002351612},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.14219155604951084},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13736106897704303},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13831142103299499},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13748008105903864},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13840220007114112},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13845980702899396},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1380873619345948},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1398809979436919},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13817874295637012},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1415796340443194},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13930849998723716},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1382785600144416},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13962158397771418},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1405406369594857},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.14256131497677416},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.14132267492823303},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1399826769484207},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1424951220396906},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.14040507399477065},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13882572995498776},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.14071027399040759},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.14136212889570743},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1384833409683779},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13894324796274304},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1377160110278055},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.14211145602166653},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13795131898950785},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1381470450432971},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13779660093132406},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13947479100897908},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1357347370358184},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13712502899579704},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13690169202163815},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13843807694502175},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13662927399855107},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1385967229725793},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13792190398089588},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1373421640601009},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13715405308175832},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.14091702899895608},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.14016691106371582},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1378888830076903},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13758336298633367},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13721647893544286},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.14038096996955574},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13941477902699262},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1365189690841362},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1371597939869389},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13678391301073134},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13742459705099463},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13696979894302785},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13762061297893524},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1385369310155511},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.14257320703472942},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.14162981789559126},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13948825490660965},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.14010654797311872},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.14025708893314004},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.14044250501319766},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.14114374201744795},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1424774390179664},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.14306695805862546},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13983892905525863},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.14055742893833667},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.14144464302808046},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13956872490234673},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1401409519603476},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13692749000620097},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1410204820567742},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1389168290188536},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13836813694797456},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13665117498021573},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13794798299204558},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13652854599058628},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1378042349824682},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1392332089599222},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.14093428093474358},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1382618979550898},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13889103091787547},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.14039573702029884},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1423256549751386},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.14010285097174346},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13853733206633478},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1393331460421905},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13855154905468225},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1364211360923946},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13931483204942197},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13682185299694538},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13685055705718696},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13819245889317244},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13887467107269913},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.14280460798181593},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13680687500163913},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13832503708545119},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13681160402484238},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1418474739184603},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13729842193424702},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13849941093940288},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1386162299895659},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13753359008114785},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13732533203437924},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13685088802594692},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1374111920595169},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13763839600142092},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13719927694182843},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13927754294127226},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13885713799390942},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1400152969872579},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1378168889787048},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1369502319721505},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13712000008672476},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13775859004817903},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13757084996905178},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13904382602777332},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1401034720474854},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13864767807535827},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13651111396029592},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.137195140006952},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13755363796371967},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13733508996665478},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13688035204540938},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13788413407746702},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13825420406647027},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.14098036708310246},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13620705693028867},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1364991719601676},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13689577102195472},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13773991505149752},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13711900694761425},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13672453293111175},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13696542102843523},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13548602210357785},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13663046702276915},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1427189289825037},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1391227840213105},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13927680091001093},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1395935210166499},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13994091795757413},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.14585228101350367},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.14516088203527033},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1410187380388379},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1411441919626668},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13859496999066323},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13789256894961},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13778296601958573},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13856451294850558},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13803742895834148},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13968992105219513},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13853536790702492},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.14392756298184395},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13904443697538227},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1387944109737873},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13943217997439206},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.16180472495034337},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14269276009872556},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.15017839695792645},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14322230697143823},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14141342393122613},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1415768200531602},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1378422660054639},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13874181301798671},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13892892200965434},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1385341959539801},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1431361670838669},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14175813703332096},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1375964469043538},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13818273099604994},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1375288019189611},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1417094769421965},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1405665249330923},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13794665003661066},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14421763294376433},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13638024008832872},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1361364749027416},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1367932299617678},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13694007392041385},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13686968199908733},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13621060305740684},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1355359760345891},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13661688100546598},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13588404492475092},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13810385297983885},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1376657970249653},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13677933404687792},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.14031114906538278},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13859935803338885},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13674805604387075},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13680291804485023},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13944549509324133},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.14085626602172852},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.14210313092917204},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13823599100578576},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13813765696249902},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1376279960386455},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13782550499308854},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13747100299224257},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13732361898291856},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13757557899225503},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13837086095009},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1445747090037912},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.14195285993628204},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13717405998613685},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13743074797093868},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1370809959480539},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13734540005680174},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13820330903399736},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13841925200540572},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1372573949629441},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.14128515496850014},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13795362308155745},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13786172203253955},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13800613197963685},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1373675400391221},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.14250443899072707},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1433556160191074},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.14047952299006283},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13961502094753087},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13731329992879182},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13698697194922715},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13766265101730824},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1362835600739345},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13744170893914998},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13697977806441486},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13739677506964654},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1369075640104711},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13745453290175647},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1402287760283798},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13757051900029182},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13703543203882873},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13724684598855674},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13831554795615375},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13977188407443464},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13752472400665283},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1379429439548403},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13750248192809522},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.14088965801056474},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1370979379862547},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13854038796853274},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1398550090380013},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.14037735294550657},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.14012834895402193},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1389279910363257},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13742989709135145},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1379236279753968},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1398924180539325},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13661162101197988},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.14115478203166276},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13832174998242408},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13699224102310836},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13772387499921024},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13712948700413108},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13799288601148874},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13904812501277775},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1401894629234448},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1380186149617657},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1385902410838753},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.14002516691107303},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13776300894096494},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13817690999712795},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1394627570407465},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1370554791064933},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13949135108850896},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.14436666097026318},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.14012452098540962},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.14047335204668343},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13880884798709303},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13981929305009544},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1385541040217504},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13936764001846313},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1394543619826436},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1402264810167253},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13926147297024727},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13833962299395353},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13967352989129722},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13849538401700556},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1375281810760498},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13806419004686177},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1409981510369107},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13761228800285608},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13921822106931359},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.1382816950790584},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13753129600081593},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13816390500869602},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.1392059080535546},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.1379440960008651},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13794981699902564},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13772086007520556},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13989191805012524},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.14458471804391593},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.14005847892258316},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13771882595028728},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13853219198063016},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.138786907074973},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1400509539525956},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13846667006146163},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13943733007181436},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.14030118996743113},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13748857600148767},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13821901893243194},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1404173980699852},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1431734060170129},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.14321603591088206},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.14328364201355726},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.14193540695123374},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.14384361600968987},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.14234763605054468},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.14265693409834057},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.1420852169394493},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.14339659304823726},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.14251401694491506},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.14692906907293946},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.14623829000629485},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.14535869099199772},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.14022700302302837},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.1385955709265545},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.140739397960715},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.14307037403341383},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.14561818400397897},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13759741000831127},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.1393160440493375},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13878220797050744},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.1381588359363377},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13848471397068352},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.14165436394978315},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13976353802718222},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13783032400533557},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13988729007542133},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13841382204554975},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13918088097125292},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13780971604865044},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13849914097227156},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.14033981296233833},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13722217897884548},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13794633105862886},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13763866596855223},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13949318404775113},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13930403196718544},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.14068567799404263},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.14157220104243606},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.1404193009948358},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13718111300840974},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13903115305583924},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13918588100932539},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.14043451996985823},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13938337902072817},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13796656706836075},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13971219293307513},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13808679301291704},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.1366685670800507},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13788816204760224},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13611167890485376},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13724051299504936},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.14053448499180377},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.137007339973934},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.14113931404426694},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13870352192316204},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13756289496086538},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13772316405083984},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13725496095139533},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1374566670274362},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1368801319040358},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.14321264997124672},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13925116299651563},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13904693198855966},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1381934400415048},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13758375495672226},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13853228196967393},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13665784802287817},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13909585308283567},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13965299201663584},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.14101472101174295},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13849307992495596},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13700512505602092},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13811385200824589},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13743005704600364},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.14056675601750612},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13796322199050337},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13676671998109668},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13779319496825337},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.14024721004534513},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.1374699609586969},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.137418866972439},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13670346199069172},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13750571908894926},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13765956496354192},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.14097042893990874},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.14260616898536682},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.14100497297476977},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.14016222092323005},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.13875909510534257},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.13965240109246224},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.1377754620043561},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.13817611802369356},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.13953812804538757},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.13768460194114596},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.13836300699040294},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.13922518398612738},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.13874931703321636},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.13921981398016214},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.13815621100366116},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.13907998404465616},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.14085024397354573},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.13889405701775104},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.13967588497325778},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.13993449707049876},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.13982793898321688},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.13865064294077456},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1391103200148791},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.13852536992635578},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1388113929424435},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.14155352592933923},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1411337930476293},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.14047594706062227},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.13975692505482584},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.13943896302953362},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.13842003303579986},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.1414792980067432},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.13921649800613523},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.13987233105581254},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.13895613292697817},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.13840587704908103},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.14262361102737486},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.14015049999579787},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.1388153110165149},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.13840021705254912},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.1383208379847929},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.13838523894082755},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.1396279860055074},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.1377360470360145},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.1382441259920597},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13978453702293336},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13952317007351667},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.13917691400274634},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.13811905204784125},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.13921763096004725},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1393868059385568},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.1398479660274461},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.1389412049902603},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.14271242602262646},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13804067601449788},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.1397913299733773},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.1378191530238837},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.1377423000521958},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13674239499960095},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13774005603045225},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13749452796764672},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13793314492795616},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.14590806502383202},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.14215017796959728},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13741834496613592},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13886334002017975},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13959881197661161},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13817889406345785},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.1386084440164268},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13919180200900882},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.1387751450529322},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.1419330940116197},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.1387969059869647},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.1382167749106884},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.13904396700672805},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.13999189506284893},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.14066585095133632},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.14179597795009613},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.1407135089393705},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.1392303539905697},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.13807325705420226},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.1393125270260498},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.13860169204417616},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.1391392849618569},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.1396024179412052},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.1387516320683062},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.13871736801229417},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.14270953100640327},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.13930641603656113},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.1395841840421781},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.1401224680012092},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.13981203909497708},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.14254054601769894},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.14356870402116328},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.14026015403214842},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.1419404069893062},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.1400071020470932},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.13969861692748964},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.14007908594794571},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.14003514498472214},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.14320469496306032},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.14138002193067223},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.13943783997092396},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.1437481379834935},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.14121702895499766},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.1411044190172106},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.14038088999222964},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.1417770319385454},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.14472212397959083},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.1411301960470155},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.14358277898281813},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.1412238710327074},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.1410421619657427},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.14084634801838547},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.14165389304980636},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.14138176594860852},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.14212181500624865},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.15710711700376123},{"candidates":13,"kind":"choice","padded_tokens":2119,"seconds":0.14043567201588303}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-arc-easy-standard-v2","finished":1792205972,"gpu":0,"id":"7e4ed737ed0c83699e490690","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","model_revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd","n":2376,"notes":"Full ARC-Easy test split (2376 examples); FP32, 1024-token input ceiling, raw summed answer-token log likelihood, zero-shot Question/Answer prompt. Character-normalized accuracy reported separately as acc_norm. Native masked-padding adapter uses unpadded row forwards. Distinct from the earlier token-normalized pilot.","peak_gpu_bytes":841644544,"profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Easy (full test split, raw accuracy)","catalogue_id":"arc-easy","choice_score":"sum_log_likelihood","contract":"Question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"sub1b-arc-easy-full","license":["cc-by-sa-4.0"],"limit":2376,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / full sub-1B standard evaluation","parent_task":"reading","priority_models":["SlayerLab/Slayer149-ARC"],"protocol_version":"codesota-sub1b-arc-easy-standard-v2","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_standard.py","selection":"Full pinned test split: 2376 examples; raw summed conditional answer likelihood, zero-shot Question/Answer prompt; separate character-normalized diagnostic","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":87.13233877101447,"measurement_version":"gpu-slot-v1"},"runner_sha256":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","scope":"cf50d4b44de811db4aed984c74f66a1eab461af452494ae6c905f9c80d3ffc25","score":0.5934343434343434,"scoring":"argmax raw summed conditional answer-token log likelihood; zero-shot Question/Answer prompt; no chat template or added BOS","seconds":84.23893190396484,"seed":42,"selection":"full pinned ARC-Easy test split; all 2376 examples in source order","status":"completed","task":"sub1b-arc-easy-full","worker":"white-gpu0"}