{"acc_norm":0.6670875420875421,"acc_norm_definition":"argmax summed conditional log likelihood / Python character length of answer text, excluding leading delimiter","batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":176,"seconds":0.3089594249613583},{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.13527000998146832},{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.13301810005214065},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.1357149500399828},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.13516611605882645},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.1331578209064901},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.1380391129059717},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.13491365499794483},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.13362501305527985},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.13415872806217521},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.1329197681043297},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.13442848098929971},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.1340868240222335},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.1354641909711063},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.13575549505185336},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.1384617609437555},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.13709988095797598},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.13635871990118176},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13686108600813895},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13560162798967212},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13384957192465663},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13555454998277128},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13498701201751828},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13484125002287328},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.1339573620352894},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.132781479973346},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13317424105480313},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13278430490754545},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13374649896286428},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.1357272119494155},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.1405773259466514},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.1385121849598363},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13647304300684482},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13470886298455298},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13424748310353607},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13436252798419446},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13474888796918094},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13585935800801963},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13393548096064478},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.133262507035397},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13560853095259517},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13790755707304925},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13701484294142574},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13584270793944597},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13568889000453055},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.1348160640336573},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13426628801971674},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13700814102776349},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13358399597927928},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.1338306670077145},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.1338835150236264},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13461579999420792},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13619926199316978},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13978069997392595},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13587965606711805},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13495378999505192},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.1353529649786651},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13581722101662308},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13489694404415786},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13632224197499454},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13615151401609182},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.1346751410746947},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13733562093693763},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13782379205804318},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13736923399847},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13568394188769162},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13705260411370546},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13808861491270363},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13577112404163927},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13621468201745301},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13475689303595573},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1345999799668789},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13439166196621954},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13569288910366595},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1355542199453339},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13762050308287144},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13512693298980594},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1366315990453586},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13743913499638438},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1365311909466982},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13721115898806602},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13566798204556108},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13511011190712452},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13681275607086718},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1353400700027123},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13474773603957146},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13842282793484628},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13606138504110277},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13678751897532493},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1372923309681937},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13612986297812313},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13669455598574132},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13804443296976388},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13766206102445722},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13826346094720066},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1360410270281136},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1393321939976886},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13794620998669416},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1364210060564801},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13799022196326405},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13641646702308208},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13601587002631277},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13783665595110506},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1363646099343896},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13765824295114726},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13638464792165905},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13927845400758088},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13735751202329993},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13688594300765544},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13864511298015714},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.14117144304327667},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1370375349652022},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1376320249401033},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.137313078972511},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13764288497623056},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13881585106719285},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13778794498648494},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13982093497179449},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1408333129948005},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.14311738207470626},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13782205805182457},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13742855400778353},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13963948702439666},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.14552271598950028},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1376898109447211},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13872435095254332},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13849703699816018},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13575784000568092},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13540257699787617},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1372596799628809},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13621977099683136},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13737824093550444},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1357424610760063},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13861574803013355},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13692082790657878},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1370247210143134},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13495945103932172},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13573399500455707},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13622777606360614},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13555636396631598},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13497500005178154},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13969957898370922},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13731812790501863},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.14078086509834975},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13683099998161197},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13574265106581151},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13610792206600308},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13548830698709935},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13611953402869403},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13600728497840464},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13503918889909983},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1355093460297212},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1362010360462591},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1374090180033818},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1387508399784565},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13989180698990822},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1390693939756602},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.14116862800437957},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1396792819723487},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13983147498220205},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.14062927302438766},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13931080407928675},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.14065559196751565},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.14146447996608913},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.14146291697397828},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13927493803203106},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13804321095813066},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13945684698410332},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13660014991182834},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13753902993630618},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1355206579901278},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1372316179331392},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13805749698076397},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13790445204358548},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1359474229393527},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13724967103917152},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.136757202912122},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13941664202138782},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13794669101480395},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13647638901602477},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13826809893362224},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13768603501375765},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13738164794631302},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13654173188842833},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13758495799265802},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13556725590024143},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13518050289712846},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13730859197676182},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13817946496419609},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13650147605221719},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1363989650271833},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.14053353399503976},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.14005457109306008},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13921751000452787},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13638239493593574},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13779211207292974},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13617773295845836},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13738929200917482},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13564759399741888},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13612500403542072},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1406378879910335},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13642475300002843},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13602641108445823},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13816559908445925},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13715516403317451},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1368397769983858},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13615266501437873},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1367664400022477},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1354316920042038},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13522081798873842},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1359897720394656},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13595263299066573},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13522355305030942},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13656649691984057},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13532501296140254},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1354973439592868},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13557119097094983},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13769370899535716},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1372716419864446},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13802817300893366},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13987085805274546},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13838232401758432},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13631860504392534},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13500102795660496},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1363240759819746},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13492093898821622},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1355107689741999},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1367991209262982},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13543770206160843},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13578085193876177},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1371471299789846},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13660323596559465},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13880295702256262},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1359407000709325},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13671689806506038},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.137571120983921},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13760937203187495},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13762911793310195},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1388515679864213},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13882711296901107},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13765285292174667},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.14329330995678902},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14155203301925212},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1420440310612321},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14032493601553142},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1379671780159697},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1387268960243091},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13949530804529786},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13764448696747422},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14193144999444485},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13969406904652715},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1417059099767357},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1400663930689916},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13819311093539},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14220650296192616},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14110117289237678},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1415995709830895},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14055644697509706},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.16674061899539083},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14695062895771116},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14009023702237755},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13899765105452389},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1405014239717275},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13974963198415935},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13836024201009423},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13841060595586896},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.14370645000599325},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1408487519947812},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13877877197228372},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1378098949790001},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13673026207834482},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.14196106500457972},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1368616579566151},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13719269493594766},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1366973320255056},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13646023895125836},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1369847769383341},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13723268907051533},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13796054699923843},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1369643990183249},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13763857702724636},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.14159339992329478},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13962705410085618},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13662160898093134},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13789744896348566},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13721407402772456},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13743846293073148},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13707027700729668},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13820201705675572},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.137140606995672},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.14167527307290584},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1375959060387686},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.14146885799709707},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.14057924901135266},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13659350795205683},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13940528000239283},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13817275199107826},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.14001408498734236},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13718745589721948},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13924822804983705},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13857093395199627},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13763265509624034},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13776015304028988},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13718127401079983},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13810154900420457},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1365395070752129},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13715889095328748},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13667717308271676},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.14371411397587508},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1368196289986372},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1393774279858917},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1360287939896807},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1378380679525435},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13866849604528397},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1369509039213881},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1363779860548675},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13713256199844182},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13896493997890502},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1393826080020517},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13828369905240834},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13628543308004737},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13700826093554497},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13780875399243087},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.137470513000153},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13734304602257907},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.138553983066231},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13769921998027712},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13950976496562362},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.14153512101620436},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13859156297985464},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13772838306613266},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13673759601078928},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13750528800301254},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13755926897283643},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13762627309188247},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1376645639538765},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.14176178397610784},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13854533690027893},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13771127199288458},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13817618798930198},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1379001239547506},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13662857306189835},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13877924298867583},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.14135961409192532},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.14257838705088943},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13940353703219444},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1382714060600847},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13698666100390255},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13762019097339362},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13729441503528506},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13727958593517542},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13852627202868462},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13743660901673138},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13674321700818837},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.14110448805149645},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13650120503734797},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13582506496459246},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13678984297439456},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13689888699445873},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.14023847389034927},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1392878310289234},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13657648593652993},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1372350730234757},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13720018800813705},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1386818620376289},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13623347599059343},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1377287550130859},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13715065701398998},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13876090897247195},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.1409269979922101},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13812802906613797},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13731485302560031},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.137695551966317},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13948082097340375},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13775510399136692},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.14024195994716138},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.138241570093669},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.1367521439678967},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.1380481399828568},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13679523300379515},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13864423101767898},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13605759793426841},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.14232319104485214},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1372729039285332},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1390972959343344},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13640884298365563},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13586206396576017},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1371432120213285},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13656489504501224},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13718027097638696},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13894023303873837},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13921085803303868},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1390219050226733},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.1406929810764268},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.1408742390340194},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.1394790270132944},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13772810297086835},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.1381037529790774},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.1388011040398851},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13678931293543428},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.14482813200447708},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.1429180899867788},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.14275095891207457},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.14233295898884535},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.14223032898735255},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.14246249094139785},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.1435762170003727},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.14414214296266437},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.14198964892420918},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.14201327308546752},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.14209545601624995},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.14211109594907612},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13864562404341996},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.14007878699339926},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.14632132498081774},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.14350440306589007},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.1428527480456978},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.14043406897690147},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.14139346801675856},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13902504101861268},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13752439303789288},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.14156186196487397},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.14161944901570678},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13853078905958682},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13744148903060704},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13827324996236712},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13844862708356231},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.14043760497588664},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.14023556897882372},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.1376217040233314},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.1371508869342506},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13915834901854396},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13851776602678},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13916157698258758},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13800123205874115},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13843950000591576},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.14028143393807113},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.14318296406418085},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13910894806031138},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.14107961195986718},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.1397664740215987},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13681829709094018},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.14210860000457615},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.14119121001567692},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.14114937209524214},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13823658099863678},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13734955806285143},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.14274237304925919},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13716683606617153},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13822314597200602},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13661563908681273},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13923853007145226},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.14089600008446723},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13844108197372407},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13759920292068273},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13658875902183354},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.14215332397725433},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13770269602537155},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.1379581419751048},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13754391996189952},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13850372901652008},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13683420699089766},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13756135199218988},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13673154497519135},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.1375152460532263},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13622333796229213},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13974761799909174},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13903154304716736},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.14343048597220331},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.140342547907494},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13921313197351992},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.1386512949829921},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13783337897621095},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.1383636990794912},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.14022388693410903},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13789223902858794},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.13859919703099877},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.13715533504728228},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.13707328191958368},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.1358077919576317},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.13805761700496078},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.1359387970296666},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.13779407599940896},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.14610482205171138},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.13992670294828713},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.13861488597467542},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.13936194905545563},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.13742505805566907},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.137770873028785},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.1387948520714417},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.13771045999601483},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.1384765279944986},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.14017052692361176},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.14301603310741484},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1418401210103184},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.13967681594658643},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1382057439768687},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.13892432407010347},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.14048928208649158},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.14172681793570518},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1419568380806595},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.13903281604871154},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.13908603496383876},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.13807115296367556},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.1383207889739424},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.13952792901545763},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.13812119595240802},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.13874958804808557},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.14428276498802006},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.13930787902791053},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.13927688100375235},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.13997822802048177},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.13874555996153504},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.1385497460141778},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.1411665150662884},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.13920244202017784},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.13554438191931695},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.1392195540247485},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.14034019294194877},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.1407528129639104},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13769257708918303},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13757485698442906},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.13771137199364603},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.13893514405936003},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1392844649963081},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.13820190692786127},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13967698602937162},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13761951099149883},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.1385506569640711},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.14033897197805345},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.14105595799628645},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13730556599330157},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.138106127968058},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13965207000728697},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.14186719001736492},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13933807495050132},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13851607195101678},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.14010478497948498},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13952743797563016},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13911168300546706},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13867648190353066},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.1391184349777177},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.1458523420151323},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.14183899899944663},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.14145592402201146},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.14155197306536138},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.13969818607438356},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.13784940994810313},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.1379317829851061},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.138041818048805},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.14181816997006536},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.13817077898420393},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.1378487580223009},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.137365086004138},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.13801441702526063},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.13931724708527327},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.13773059810046107},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.13835583301261067},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.13730688788928092},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.14148209302220494},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.13841876992955804},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.1411837360356003},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.13892407296225429},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.14012145507149398},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.1379270339384675},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.13866212510038167},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.14217555604409426},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.14579873101320118},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.14207444700878114},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.1410055939340964},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.1405195479746908},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.14038342505227774},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.14047980296891183},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.14122039498761296},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.1402317809406668},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.14361419796478003},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.14329773792997003},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.1404978969367221},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.14166723692324013},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.14207483793143183},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.14220923895481974},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.1410589530132711},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.1443875200347975},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.140928901033476},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.14135571802034974},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.14065748499706388},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.14069910300895572},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.1422695709625259},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.14172826195135713},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.14504168997518718},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.14200245204847306},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.14819148310925812},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.15425389597658068},{"candidates":13,"kind":"choice","padded_tokens":2119,"seconds":0.13960612507071346}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-arc-easy-standard-v2","finished":1792205869,"gpu":0,"id":"da7edd3fa6604be87c71349a","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149-ARC","model_revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17","n":2376,"notes":"Full ARC-Easy test split (2376 examples); FP32, 1024-token input ceiling, raw summed answer-token log likelihood, zero-shot Question/Answer prompt. Character-normalized accuracy reported separately as acc_norm. Native masked-padding adapter uses unpadded row forwards. Distinct from the earlier token-normalized pilot.","peak_gpu_bytes":841644544,"profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Easy (full test split, raw accuracy)","catalogue_id":"arc-easy","choice_score":"sum_log_likelihood","contract":"Question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"sub1b-arc-easy-full","license":["cc-by-sa-4.0"],"limit":2376,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / full sub-1B standard evaluation","parent_task":"reading","priority_models":["SlayerLab/Slayer149-ARC"],"protocol_version":"codesota-sub1b-arc-easy-standard-v2","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_standard.py","selection":"Full pinned test split: 2376 examples; raw summed conditional answer likelihood, zero-shot Question/Answer prompt; separate character-normalized diagnostic","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":86.7796474751085,"measurement_version":"gpu-slot-v1"},"runner_sha256":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","scope":"cf50d4b44de811db4aed984c74f66a1eab461af452494ae6c905f9c80d3ffc25","score":0.6658249158249159,"scoring":"argmax raw summed conditional answer-token log likelihood; zero-shot Question/Answer prompt; no chat template or added BOS","seconds":83.8724024950061,"seed":42,"selection":"full pinned ARC-Easy test split; all 2376 examples in source order","status":"completed","task":"sub1b-arc-easy-full","worker":"white-gpu0"}