{"acc_norm":0.6258417508417509,"acc_norm_definition":"argmax summed conditional log likelihood / Python character length of answer text, excluding leading delimiter","batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.29751766100525856},{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.11615649203304201},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.11515539500396699},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.11525060201529413},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.11633601598441601},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.11664502299390733},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.1166567848995328},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.11541586997918785},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.1148280439665541},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.11494403099641204},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.11516283906530589},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.11830072000157088},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.1167210239218548},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.11506786104291677},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.11867500795051455},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.11505742196459323},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.1149308760650456},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.11489570094272494},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.1151250780094415},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.11521943402476609},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.1149516950827092},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.1151234059361741},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.11540726397652179},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.11533612199127674},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.11556210403796285},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.11503006203565747},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.11513701104559004},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.11577073205262423},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.11507046595215797},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.11583460099063814},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.11885915009770542},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.11586507805623114},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.1169752279529348},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.1148431720212102},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.11568317899946123},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.11696206300985068},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.1172804479720071},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.11645112000405788},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.11582790804095566},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.11538787803146988},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.11553485202603042},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.11558712006080896},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.11604483309201896},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.11579855403397232},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.11572734103538096},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.11578035994898528},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.11579581792466342},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.11680822703056037},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.11785027896985412},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.11942841194104403},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.11564233200624585},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.1155800869455561},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.11532147403340787},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.11541374609805644},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.11574389203451574},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.11591911897994578},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.11571380600798875},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.11564022896345705},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1161841630237177},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.11597313999664038},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.11608140205498785},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.11591208598110825},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.11866880604065955},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.11861861194483936},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.12227995891589671},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.11957105807960033},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.12399974605068564},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1223878200398758},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1166328089311719},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.11568422091659158},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.11635403998661786},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.11633186903782189},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.11623071995563805},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1157576380064711},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.11608760291710496},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.11566668807063252},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.11562663305085152},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.11584383808076382},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11649308900814503},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1167191700078547},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11625807103700936},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11643512104637921},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11599292606115341},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11590869899373502},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11708627606276423},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11852541798725724},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11702834698371589},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11551292100921273},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11560623603872955},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11597873002756387},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11602265201508999},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11580211098771542},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11597299901768565},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11613452108576894},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11710758495610207},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11763782403431833},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11693175695836544},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11599297693464905},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11587248102296144},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11543795198667794},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1158999539911747},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11839528498239815},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11642017308622599},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11721353302709758},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11784163396805525},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11554199608508497},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11596821004059166},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11568608391098678},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11579481698572636},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11579764191992581},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11611085699405521},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11580130993388593},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11549888493027538},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11559543502517045},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1158022410236299},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11671161698177457},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11629711394198239},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11627185600809753},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1166088649770245},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11759657599031925},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11616269301157445},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11616487707942724},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11918260401580483},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11900733702350408},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11813404900021851},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11728805094026029},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11634643492288888},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11616851389408112},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11614831606857479},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11788461403921247},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11738851806148887},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1163618239806965},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11688445904292166},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11670061596669257},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11665720504242927},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11670089699327946},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11752739804796875},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11603272100910544},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11596818000543863},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1161391589557752},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11770673200953752},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11834711593110114},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11621739401016384},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11614774505142123},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11614372802432626},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11624719097744673},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11618546594399959},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11624881299212575},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11596248997375369},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11638847400899976},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11665084294509143},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11659252399113029},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11836927698459476},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11882435600273311},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11862126796040684},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11589527397882193},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11588453501462936},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11571108095813543},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11669734003953636},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11941870301961899},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1169319769833237},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1160570359788835},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11622889596037567},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11622925696428865},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11642808793112636},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11649390100501478},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11659548000898212},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11820578202605247},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11990346806123853},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11999979696702212},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1203424169216305},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.12156718003097922},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11981000297237188},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11984687193762511},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11998445901554078},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.12026886898092926},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12015046796295792},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12118030898272991},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1207426420878619},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12009060708805919},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1200981610454619},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12245498597621918},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12231414299458265},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12133916502352804},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11787061800714582},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11730436200741678},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11866025999188423},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12020747503265738},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12046090804506093},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11609639902599156},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11631526693236083},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11675843398552388},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11622879607602954},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11614587099757046},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11628968000877649},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1186863190960139},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11787321302108467},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11619520303793252},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11624579795170575},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11632003600243479},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1171355169499293},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11597162601538002},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11599871702492237},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11647107801400125},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11615928693208843},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11737876001279801},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11594273301307112},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11594602896366268},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11639853299129754},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11637186398729682},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11636398790869862},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11775344901252538},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1176430230261758},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1190632910002023},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11872358794789761},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11627580400090665},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11598325800150633},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11670796002727002},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11642539291642606},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11689909710548818},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11624527699314058},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11626882094424218},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11703069100622088},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11645928595680743},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11566040699835867},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11614970897790045},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11612106603570282},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11698782199528068},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11631292302627116},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1160271909320727},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11644374707248062},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11626840010285378},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11927637900225818},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11876255099195987},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11665787594392896},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1162882469361648},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11699672695249319},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11672754597384483},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1165938270278275},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1163181730080396},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11798183503560722},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11723150592297316},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1178508810698986},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1176317430799827},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11641784803941846},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11617583699990064},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11660539801232517},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11674583097919822},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1163027340080589},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11621812602970749},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11769633297808468},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11865180404856801},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1163544199662283},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11641683697234839},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11637855600565672},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11641225800849497},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11665065202396363},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11852358502801508},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11635475093498826},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11596464400645345},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11642211698926985},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1164855050155893},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11669623793568462},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11697984603233635},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11638350505381823},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11641721799969673},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11671570397447795},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11643225594889373},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11740724393166602},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11946433898992836},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1196758629521355},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1189988210098818},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11785033997148275},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11689260508865118},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11728285090066493},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11830061895307153},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11633846105542034},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11600777402054518},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11599990003742278},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11618224903941154},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11588810104876757},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1161873689852655},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11632575700059533},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11634867009706795},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11638262297492474},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11671408207621425},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1166017510695383},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11823344393633306},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11812456091865897},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11665398895274848},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11666825599968433},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1164661890361458},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11814689298626035},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.116207076003775},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1156438059406355},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11564643995370716},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11583740694914013},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11569908796809614},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11693429097067565},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11811558401677758},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11838105996139348},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11730418202932924},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1165362490573898},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11633736896328628},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11640565597917885},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11730365105904639},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11909639299847186},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11682721192482859},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11615366605110466},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11653272295370698},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11660950595978647},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11631245305761695},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11607692297548056},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11605087493080646},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11589698703028262},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11585390695836395},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11619301897007972},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11650911800097674},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11655254999641329},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1163956270320341},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11671978200320154},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11629271507263184},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1165186770958826},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11636536102741957},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11786554893478751},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11796869104728103},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11694287799764425},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11777307605370879},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11716049304232001},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11799815599806607},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11793167097494006},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1165429720422253},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11627362004946917},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11600280401762575},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.116358297993429},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11738490196876228},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11661400506272912},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11612645594868809},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1162316519767046},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11664167593698949},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11662005598191172},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11672738508787006},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11718888697214425},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11907116591464728},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11819938092958182},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11618954304140061},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11616306391078979},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11635581299196929},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11642953008413315},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11609778297133744},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11631621897686273},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11645411606878042},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11667250306345522},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11641684698406607},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1167593850987032},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1165139980148524},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.11648838000837713},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.11623118992429227},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.11794442497193813},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.1182790189050138},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.11805904901120812},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.12051938602235168},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.11844641203060746},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.11617175990249962},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.11626709799747914},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.11656768794637173},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.11656504403799772},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.11646815296262503},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.11674719292204827},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.11659127206075937},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.11674476903863251},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.11686286795884371},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.11549359501805156},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.11514182901009917},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.11552396207116544},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.11555826594121754},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.11548848601523787},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.11520915501751006},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.11476106895133853},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.11878243798855692},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.11716141598299146},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1150729819200933},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11546378908678889},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11535870400257409},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11515522503759712},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11537589610088617},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11593818396795541},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11555825592949986},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11719332507345825},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11721166898496449},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11691451398655772},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11573824193328619},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11569729598704726},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.11553740699309856},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.11559822107665241},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.11551724001765251},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.11589803895913064},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.11854178900830448},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.11756443697959185},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.11572075902950019},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.11521245096810162},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.11518651305232197},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.11536919302307069},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.11523019406013191},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.1155038649449125},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.11576597299426794},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.11582323000766337},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.11546409002039582},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.11588983400724828},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.11556982691399753},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.11614170391112566},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.1168490129057318},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.11691710993181914},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.1149952569976449},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.11844531900715083},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.11527718091383576},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.11722333100624382},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.11827683506999165},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.1168471589917317},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.11565907299518585},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.11621154402382672},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.11525585292838514},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.11521551595069468},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.1152992929564789},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.11546714592259377},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.11493460310157388},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11520091001875699},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.1145901910495013},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11536278098355979},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11510388902388513},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11565576796419919},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11552905198186636},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11485140805598348},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11438297491986305},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.1144482959061861},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11652129201684147},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11570184293668717},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11438122089020908},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11465300794225186},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11491772101726383},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11475472804158926},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1147238400299102},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.11515861097723246},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.11478181800339371},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1148595130071044},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.11470657808240503},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1151363190729171},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.11527192196808755},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.11786288407165557},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.12088547903113067},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.11844867607578635},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.1170395869994536},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.11671039403881878},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.11720017797779292},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.11820660310331732},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.1177573159802705},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.11793964600656182},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.11798084294423461},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.11766908201389015},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.11765684897545725},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11793666100129485},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11793114093597978},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11747697391547263},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11787725007161498},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11654170998372138},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11496101296506822},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11498913494870067},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11601381504442543},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11474316602107137},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11474807595368475},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11475314502604306},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11523317906539887},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11631443491205573},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11646922503132373},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11549233295954764},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11598425998818129},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.1153273560339585},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11603550601284951},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11673474893905222},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11815387499518692},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11712838395033032},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11600964795798063},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11706751002930105},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11679696594364941},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11964409402571619},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11880498006939888},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11803230899386108},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11624048801604658},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11649744701571763},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.11608495796099305},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.11572859401348978},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.11690118000842631},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.11532758607063442},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1147260150173679},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.11455601698253304},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.11515098600648344},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.11474001000169665},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.11434927198570222},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.11420787801034749},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.11505388596560806},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.11459814594127238},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.11489821597933769},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.11493743897881359},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.11575234902556986},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.11584006203338504},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.11558321199845523},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.11529036599677056},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.11543711007107049},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.11687984992749989},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.11900581500958651},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.11896553996484727},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.11578861600719392},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.1161810380872339},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.11615397699642926},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.11566180898807943},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.11559389194007963},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.11551323195453733},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.11507196899037808},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.1152949349489063},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11636307602748275},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11536314198747277},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11546899005770683},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11514449503738433},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11500328104011714},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11543438502121717},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11528860300313681},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11545666703023016},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11556400707922876},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11701675597578287},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11596932297106832},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11497429804876447},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11484733002725989},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11480230710003525},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11506652901880443},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.11507412302307785},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.1150112560717389},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.11571408703457564},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.11598511203192174},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.11519316502381116},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.11558122897986323},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.11617144895717502},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.1158851149957627},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.11588896298781037},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.11554488097317517},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.11574625608045608},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.1153573920018971},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.1164116469444707},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.11729905207175761},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.11549176194239408},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.11593386600725353},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.11614725389517844},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.11598602298181504},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.11548138305079192},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.11735055805183947},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.11846999498084188},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.11646788206417114},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.11688195401802659},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.11650163494050503},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.11702021106611937},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.11773509497288615},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.11703478894196451},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.11744378099683672},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.11715199006721377},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.11733869393356144},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.11724096292164177},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.11949720291886479},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.11810115899425},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.1200633579865098},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.12235712399706244},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.12257994792889804},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.12291016394738108},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.12114579498302191},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.11948764196131378},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.12522192392498255},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.1217614219058305},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.1198969460092485},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.12880933401174843},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.1223266760352999},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.1219268610002473},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.12086608307436109},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.12246684706769884},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.12259569705929607},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.12189252709504217},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.12322237598709762},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.11985548899974674},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.1191075739916414},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.119700358947739},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.12008799193426967},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.12187270901631564},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.12016659893561155},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.12137758592143655},{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.12069519399665296},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.12059044907800853},{"candidates":13,"kind":"choice","padded_tokens":2080,"seconds":0.10509475599974394}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-arc-easy-standard-v2","finished":1792205993,"gpu":1,"id":"a355f6d6302b0acac66e9d1a","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer139-1.02","model_revision":"c4298d757ba39cba19dd768777da9fadf319ff61","n":2376,"notes":"Full ARC-Easy test split (2376 examples); FP32, 1024-token input ceiling, raw summed answer-token log likelihood, zero-shot Question/Answer prompt. Character-normalized accuracy reported separately as acc_norm. Native masked-padding adapter uses unpadded row forwards. Distinct from the earlier token-normalized pilot.","peak_gpu_bytes":804225024,"profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Easy (full test split, raw accuracy)","catalogue_id":"arc-easy","choice_score":"sum_log_likelihood","contract":"Question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"sub1b-arc-easy-full","license":["cc-by-sa-4.0"],"limit":2376,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / full sub-1B standard evaluation","parent_task":"reading","priority_models":["SlayerLab/Slayer149-ARC"],"protocol_version":"codesota-sub1b-arc-easy-standard-v2","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_standard.py","selection":"Full pinned test split: 2376 examples; raw summed conditional answer likelihood, zero-shot Question/Answer prompt; separate character-normalized diagnostic","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":74.03766193194315,"measurement_version":"gpu-slot-v1"},"runner_sha256":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","scope":"cf50d4b44de811db4aed984c74f66a1eab461af452494ae6c905f9c80d3ffc25","score":0.6590909090909091,"scoring":"argmax raw summed conditional answer-token log likelihood; zero-shot Question/Answer prompt; no chat template or added BOS","seconds":71.06391803000588,"seed":42,"selection":"full pinned ARC-Easy test split; all 2376 examples in source order","status":"completed","task":"sub1b-arc-easy-full","worker":"white-gpu1"}