{"acc_norm":0.4877946127946128,"acc_norm_definition":"argmax summed conditional log likelihood / Python character length of answer text, excluding leading delimiter","batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.3086814059643075},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.12834142101928592},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.12723471701610833},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.1262478269636631},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.12577671906910837},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.12623939209152013},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.1302910060621798},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13008528295904398},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.1312222919659689},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13064163003582507},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13207616505678743},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13097052404191345},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13236170704476535},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13077145209535956},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13210413698107004},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13199339096900076},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13188984710723162},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13195990701206028},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13221264898311347},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.12945494800806046},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.1277160249883309},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.1304744880180806},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13023982010781765},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13000433205161244},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13267858803737909},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.12910454394295812},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1314537749858573},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.12954984395764768},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.12850983499083668},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1274867869215086},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.12727769801858813},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1275076559977606},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.12767690198961645},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1277481949655339},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.12800467293709517},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.12759704398922622},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1280478429980576},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.12790616904385388},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.12790483701974154},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13052520295605063},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.12870969797950238},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1289183869957924},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.12794536293949932},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1266563000390306},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.12738267297390848},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.12763845000881702},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.12819234293419868},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.12784356193151325},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.12744318693876266},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.12762896297499537},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1276524659479037},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.12720779702067375},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.12975237204227597},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.12884426803793758},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13005531707312912},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1288514919579029},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13126538298092782},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.12949584296438843},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1278697910020128},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.12731006799731404},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.12740792997647077},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1279201259603724},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.12763743801042438},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1276883230311796},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.12740947306156158},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.12754304299596697},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.12721927894745022},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1278581510996446},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.12772215600125492},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.127683573984541},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.12763115705456585},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.12756678694859147},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.12819203292019665},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13297860708553344},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.12641235394403338},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.12578722892794758},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.12586502404883504},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1267145770834759},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1273847270058468},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.12828541605267674},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.12959625094663352},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.12973044102545828},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1274943120079115},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1271128400694579},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.12727734702639282},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.12738985696341842},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.12762303091585636},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1276464150287211},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.12750401999801397},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13114540895912796},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1315765029285103},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.129119121003896},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1282055270858109},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.12822326109744608},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1267453460022807},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.12683035398367792},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1267040790989995},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.12714314693585038},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.12795359804295003},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1277649459661916},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.12698278797324747},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.12702171003911644},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.12712972203735262},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.12721436901483685},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.12744232499971986},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1289154510013759},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1305845530005172},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13240873499307781},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1298987950431183},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1274349510204047},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.12746157101355493},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1274334479821846},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.12743697501718998},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.12715202290564775},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1282301639439538},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.12762208003550768},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.12743868702091277},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.12713120493572205},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12749160593375564},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12766621203627437},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12783255195245147},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12897472199983895},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12723638000898063},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1302810279885307},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12944699299987406},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1275633400073275},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12740151898469776},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12778483296278864},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1279664720641449},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12858504499308765},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12753735098522156},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1274906649487093},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12722915690392256},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12742602394428104},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13079683005344123},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12947850103955716},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1291447589173913},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12900886603165418},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.12800156697630882},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.12767723202705383},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13151517801452428},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.12853002198971808},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1274398600216955},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1275938970502466},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.12752609106246382},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.12807331199292094},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.12777727795764804},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.12787443096749485},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.12751493009272963},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1277539860457182},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.12801888003014028},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.12767754297237843},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.12896857096347958},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1285501499660313},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.12748251005541533},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.12758274702355266},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.12807351199444383},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13142554101068527},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1283951309742406},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.12774652196094394},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1278258990496397},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.12993422092404217},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.12962035601958632},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.12913022097200155},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1280926379840821},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.12824258697219193},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.12785270891617984},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.12796249391976744},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1299582360079512},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.12812289397697896},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1277021790156141},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.12785978300962597},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.12819516903255135},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.12960986595135182},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13271120807621628},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1283086200710386},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.12832703394815326},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.12817687506321818},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1281720349797979},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.12824905896559358},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.12818272598087788},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.128366868942976},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.12790126004256308},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.12805496691726148},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13005375396460295},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.12780414894223213},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.12792105693370104},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1280660879565403},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1299004879547283},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1299512629630044},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13223036203999072},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13071832293644547},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1274504589382559},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.12711368198506534},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.12733240006491542},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.128241254016757},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.12714345811400563},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.12763763789553195},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1272075769957155},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.12781988899223506},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.12884850602131337},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1271014289231971},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1269048220710829},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.12777920300140977},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.12799869198352098},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.12731691997032613},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.12699696409981698},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13137944496702403},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.12934325903188437},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1282236910192296},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1278097489848733},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.12794556294102222},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.12756653607357293},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1279625539900735},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1278942780336365},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.12991332192905247},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13109935307875276},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.12991453404538333},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1283405589638278},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.12791653897147626},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.12793224793858826},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.12790439592208713},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.12793499394319952},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.12904482299927622},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1316433879546821},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1289905320154503},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1279121710686013},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.12749569304287434},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1276067110011354},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.12774079095106572},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.12781230395194143},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.12822361092548817},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1291249820496887},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.12740182899869978},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.12740204902365804},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.12746017705649137},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.12754732102621347},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1276010600849986},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.12851106689777225},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.12771137699019164},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.12822811002843082},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13297974993474782},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13042728090658784},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.12988593999762088},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1282540779793635},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1283169660018757},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1290221509989351},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.12888526497408748},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.12830374098848552},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1279261759482324},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.128047893056646},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1279471960151568},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1278659449890256},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.128031532978639},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.12882384995464236},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.12820359400939196},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.12757789704483002},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.12980621203314513},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13141228700987995},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.12804929609410465},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.12766804499551654},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1307784349191934},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13259688508696854},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1340828159591183},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1323547241045162},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13276178296655416},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1326457760296762},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1351529120001942},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1346766030183062},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13424645096529275},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13235354202333838},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13303126499522477},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13319140498060733},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13328605995047837},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13278300198726356},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1294777300208807},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.12890424102079123},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1282784539507702},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1289990870282054},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.12949753692373633},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.12847735395189375},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1284843779867515},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1287369589554146},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.12808481301181018},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.12913949904032052},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.12831669498700649},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.128129335003905},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.12866090703755617},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1285704489564523},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1283272639848292},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.12943100195843726},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1316235309932381},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.12832249596249312},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1284833849640563},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13284114107955247},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13045233697630465},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.12991884199436754},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.12837643595412374},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1281182951061055},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1282816999591887},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.12800004496239126},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1283833289053291},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1283956019906327},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1283379339147359},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.12818892707582563},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.12825465004425496},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.12894704996142536},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13192003301810473},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13063733209855855},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.12917352304793894},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13011687202379107},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.12869572103954852},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.12840285594575107},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.12829099805094302},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.12815677607432008},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.12978166702669114},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.12856712203938514},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.12868664506822824},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.12829858204349875},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.12883402896113694},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.12716386502142996},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.12806594802532345},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1283758949721232},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.12898660404607654},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13039484003093094},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.12802064302377403},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1283819869859144},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.12669787695631385},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.12699193402659148},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1267501439433545},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.12713709601666778},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.12767604098189622},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.12655501998960972},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.12648030102718621},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.12626870698295534},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.12653932091780007},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.12648029101546854},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.12674664705991745},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.1267048290465027},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.12671137193683535},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.12685074098408222},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13066068605985492},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.12852824898436666},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.12643718102481216},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.12652075593359768},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.12652781896758825},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.1266938999760896},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.12668562296312302},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.12750479101669043},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.12858574697747827},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.12827837304212153},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.12664779310580343},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.12625896802637726},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.12659325206186622},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.1267116820672527},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.12680587801150978},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.1267347460379824},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.12698128493502736},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13111626508180052},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.12693077989388257},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.12711703800596297},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.12734856002498418},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.1275690810289234},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.1269880080362782},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.12689446308650076},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.1265266869449988},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.12680797197390348},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.12629913294222206},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.12652727798558772},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.12663902691565454},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.12658183998428285},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.12641769496258348},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.12691430991981179},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.1271892129443586},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.12815100501757115},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13204812293406576},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.12900721305049956},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.12854827707633376},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.12789812497794628},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.12727226805873215},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.12711958307772875},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.12668202794156969},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.1267296449514106},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.1270619749557227},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.12716151005588472},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.1271296120248735},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.12688036693725735},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.12711747898720205},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.12683459103573114},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.12721075303852558},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.12787396903149784},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.1276162590365857},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13016888790298253},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.1271651079878211},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.1267474089981988},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.12707552092615515},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1271969770314172},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.12716521800030023},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.12777735898271203},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.12776099902112037},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.12719841895159334},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1275056320009753},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1278213820187375},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.12907327502034605},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.12951953802257776},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.12996376596856862},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.12893249304033816},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.1274067680351436},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.12898462091106921},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13124796003103256},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.1278231650358066},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.12782155198510736},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.1275632999604568},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.12764540303032845},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.12751423800364137},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.12729173398111016},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.1275278340326622},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.12787704495713115},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.12804174202028662},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.12837172800209373},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.12787561304867268},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.1293530479306355},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.12800578493624926},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.12744491000194103},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.12789988797158003},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.130584382917732},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.12967210297938436},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.12773882795590907},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.12780986004509032},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.12761323293671012},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.1287023030454293},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.12913258594926447},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.12911168800201267},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.12743517104536295},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.12812320399098098},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.12736312695778906},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.12772071408107877},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.12850818200968206},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.12773207598365843},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.128075635060668},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.12845650594681501},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.12818587094079703},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.13137650000862777},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.12992120592389256},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.1286772770108655},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.1283086099429056},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.1286149009829387},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.1278852199902758},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.12878468702547252},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.12822603597305715},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.12820145103614777},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.12825986905954778},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.12801921006757766},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.12994606303982437},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.12744827603455633},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.12780189502518624},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.12828712002374232},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.12810382794123143},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.13050757010933012},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.13355617492925376},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1291916660265997},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12846992106642574},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12814039597287774},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1284782460425049},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12848170299548656},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.1284083550563082},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12817012099549174},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.1287089359248057},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12792919296771288},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12913634406868368},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12833490897901356},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12853233702480793},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12857799197081476},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.1283385659335181},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.1281308380421251},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.129499901086092},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.13134329800959677},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12904643500223756},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12839214596897364},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12888864101842046},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12762105802539736},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12741611595265567},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12810632190667093},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.1278188870055601},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12865933403372765},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.13062121206894517},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.12880493490956724},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.12753106991294771},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.1283187490189448},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.12750410998705775},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.12712134595494717},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.12750611291266978},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.12946971494238824},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.12990300299134105},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.12748064601328224},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.12739172007422894},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.12766911799553782},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.12750304699875414},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.1276247249916196},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.12818999902810901},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.12810017098672688},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.12794832908548415},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.1279946049908176},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.12775417498778552},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.12800781906116754},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.12786746805068105},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.12901925493497401},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.1297145209973678},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13120249600615352},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.1336873690597713},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13586895598564297},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.1348206220427528},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.1363631880376488},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13770585204474628},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13459899893496186},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.1336727509042248},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.1387094430392608},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.1369847069727257},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13283117208629847},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13301141804549843},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.13074344000779092},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.12832529097795486},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.1281577890040353},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.13006246997974813},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.13243148697074503},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.13350991893094033},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.13238441001158208},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.13254836504347622},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.13408142398111522},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.13323580706492066},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.13254163204692304},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.13318878004793078},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.13149558298755437},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.13037678599357605},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.12824479094706476},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.1283712760778144},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.1306207609595731},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.13070238300133497},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.13059619499836117},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.1308159250766039},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.129939841106534},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.12900401593651623},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.13223339803516865},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.12958824599627405},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.12920854799449444},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.13105506100691855},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.13028602697886527},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.13191790995188057},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.13105498102959245},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.12898298702202737},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.13001897896174341},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.12938039808068424},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.12951578106731176},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.12968223099596798},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.12937225308269262},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.13097052311059088},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.12967824295628816},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.13625670003239065},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.13208770600613207},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.1323200489860028},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.13016707496717572},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.1299014399992302},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.13127572205848992},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.13088559510651976},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.1305212649749592},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.13240008894354105},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.13143867603503168},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.1309245970332995},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.13017784501425922},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.13041384506504983},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.13067289802711457},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.12992761807981879},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.12975050893146545},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.12999008502811193},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.13104250701144338},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.13408880797214806},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.13091229391284287},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.13001928897574544},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.1311557989101857},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.13021621701773256},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.12985293893143535},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.13010829605627805},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.12943294690921903},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.12999098701402545},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.1301069729961455},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.13029965199530125},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.13060573302209377},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.1305104549974203},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.13039762491825968},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.1305241510272026},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.12853618408553302},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.1326582790352404},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.14313636801671237},{"candidates":13,"kind":"choice","padded_tokens":2561,"seconds":0.12454010301735252}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-arc-easy-standard-v2","finished":1792205713,"gpu":1,"id":"0fccdec668aa67893d88f83b","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149","model_revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e","n":2376,"notes":"Full ARC-Easy test split (2376 examples); FP32, 1024-token input ceiling, raw summed answer-token log likelihood, zero-shot Question/Answer prompt. Character-normalized accuracy reported separately as acc_norm. Native masked-padding adapter uses unpadded row forwards. Distinct from the earlier token-normalized pilot.","peak_gpu_bytes":746600960,"profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Easy (full test split, raw accuracy)","catalogue_id":"arc-easy","choice_score":"sum_log_likelihood","contract":"Question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"sub1b-arc-easy-full","license":["cc-by-sa-4.0"],"limit":2376,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / full sub-1B standard evaluation","parent_task":"reading","protocol_version":"codesota-sub1b-arc-easy-standard-v2","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_standard.py","selection":"Full pinned test split: 2376 examples; raw summed conditional answer likelihood, zero-shot Question/Answer prompt; separate character-normalized diagnostic","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":81.5715113170445,"measurement_version":"gpu-slot-v1"},"runner_sha256":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","scope":"cf50d4b44de811db4aed984c74f66a1eab461af452494ae6c905f9c80d3ffc25","score":0.5677609427609428,"scoring":"argmax raw summed conditional answer-token log likelihood; zero-shot Question/Answer prompt; no chat template or added BOS","seconds":78.64737620996311,"seed":42,"selection":"full pinned ARC-Easy test split; all 2376 examples in source order","status":"completed","task":"sub1b-arc-easy-full","worker":"white-gpu1"}