{"acc_norm":0.593013468013468,"acc_norm_definition":"argmax summed conditional log likelihood / Python character length of answer text, excluding leading delimiter","batch_policy":{"choice_candidates":16,"code_test_workers":4,"generation_examples":8,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":176,"seconds":0.08003027201630175},{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.018544316990301013},{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.01925226696766913},{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.019480030983686447},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.028785923030227423},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.022342119016684592},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.021217030007392168},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.02127258398104459},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.021328968927264214},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.02148648293223232},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.028926294995471835},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.022091339924372733},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.02137194899842143},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.021555942948907614},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.022080239956267178},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.026885479921475053},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.022203960921615362},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.022189995041117072},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.02278640598524362},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.022099877009168267},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.02880748303141445},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.022090539918281138},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.022106347954832017},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.022627831087447703},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.022832763963378966},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.022434139042161405},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.027624267037026584},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.022479884908534586},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.023102427017875016},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.02317741501610726},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.023164662066847086},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.028162431088276207},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.02302820805925876},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.022748666000552475},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.02389364200644195},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.024611228960566223},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.034240022068843246},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.02535033703316003},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.023745083948597312},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.0231436729663983},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.028547980007715523},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.022837392054498196},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.021285717957653105},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.021699490025639534},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.02220048406161368},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.022018434945493937},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.029077226063236594},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.021418406045995653},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.021555562037974596},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.022609647014178336},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.022257530014030635},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.027607285068370402},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.02205111598595977},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.025780219002626836},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.029702382045798004},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.02824804000556469},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.02919640892650932},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.027078159037046134},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.0282170120626688},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.027686572982929647},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.033828375046141446},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.027407893911004066},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.02805523097049445},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.02681041997857392},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.0341506659751758},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.027195458067581058},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.027955665020272136},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.027778454939834774},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.03331914602313191},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.028323018923401833},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.030842568026855588},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.029016834101639688},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.03958854509983212},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.02988322905730456},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.028459245106205344},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.030856744037009776},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.0349074270343408},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.027860447065904737},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.027821736061014235},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.029011523933149874},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.034136280068196356},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.027670181938447058},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.02750289102550596},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.028729637968353927},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.03439071390312165},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.029440473997965455},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.027927342103794217},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.02801717002876103},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.03332110890187323},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.028723978088237345},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.027345377951860428},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.028562146006152034},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.02828892704565078},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.035575793008320034},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.02928209793753922},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.029263013042509556},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.0300254849717021},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.03487078798934817},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.02781406103167683},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.028607369982637465},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.029426037101075053},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.03515598992817104},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.03012593195308},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.03280981595162302},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.0368205449776724},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.032369805965572596},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.0317358230240643},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.02962788299191743},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.03579151502344757},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.028989752987399697},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.028573657968081534},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.031143169035203755},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.03608628490474075},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.028600107063539326},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.03016930294688791},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.030909432913176715},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.03640986897516996},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.030007811030372977},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.03076411096844822},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.029626009985804558},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.036887109046801925},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.029245159006677568},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.029226845945231616},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.030990062979981303},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.03541290003340691},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.02877628500573337},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.03442337503656745},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.030118118040263653},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.037085869000293314},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.030168491997756064},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.02976277400739491},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.032324912026524544},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.034437290974892676},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.030188028002157807},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.03191094996873289},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.03699232509825379},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.030047054053284228},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.030884174979291856},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.03044270200189203},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.03851101698819548},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.02910589997190982},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.030566723085939884},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.03231658705044538},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.03868798597250134},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.029469026951119304},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.029288931051269174},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.030959716998040676},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.03751981898676604},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.029770519002340734},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.030607329099439085},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.03873669705353677},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.03240727598313242},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.03250770294107497},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.031233977992087603},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.03690752596594393},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.03097262093797326},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.032277844031341374},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.031785745988599956},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.03806745901238173},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.031408442999236286},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.03205983806401491},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.031766240019351244},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.0381560439709574},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.03102763299830258},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.03285291697829962},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.039975516963750124},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.03173503198195249},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.032036344055086374},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.0323132099583745},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.03867387992795557},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.03161797404754907},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.03165664605330676},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.03301224403548986},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.03216555493418127},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.032159282942302525},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.03297605598345399},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.03890051203779876},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.03216651699040085},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.032868044916540384},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.03174646396655589},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.040460421005263925},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.03165371099021286},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.03169899492058903},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.038278723950497806},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.03230010496918112},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.032421472016721964},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.03303016698919237},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.04073499306105077},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.03271501907147467},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.031110516982153058},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.032273946912027895},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.03872469498310238},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.032440887996926904},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.03277575294487178},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.03925330995116383},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.03178899304475635},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.030918107950128615},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.03269171598367393},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.04151699098292738},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.03180085401982069},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0335322730243206},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0391220860183239},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03292410902213305},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.033422178006730974},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.032207884010858834},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03893795202020556},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03153875598218292},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03543987008742988},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03898834600113332},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0314100559335202},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03301052993629128},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03365618397947401},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03909318195655942},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03275904105976224},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.033358840038999915},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.039676308049820364},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.033039655070751905},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03186026506591588},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03326025593560189},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.041098339948803186},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03324297396466136},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03334910189732909},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.033197960001416504},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03843832004349679},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.033142265980131924},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.03310659993439913},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.03886978595983237},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.03185570705682039},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.03350418095942587},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.033651917008683085},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.04074383003171533},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.03262704506050795},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.03456236398778856},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.0410404420690611},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.03377599804662168},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.033241241006180644},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.03330952802207321},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.038771401974372566},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.03363542596343905},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.03449679201003164},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.03970529302023351},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.034467526944354177},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.03454227605834603},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.04039695207029581},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.03283270797692239},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.032190851983614266},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.033719743019901216},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.04009053099434823},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.03573658398818225},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.04197462496813387},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.047104750061407685},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.03802210499998182},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.03992142609786242},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.044907673029229045},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.03646457102149725},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.03964857698883861},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.047013981034979224},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.038736917078495026},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.041392078972421587},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.047606895910575986},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.0372066639829427},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.040901573025621474},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.044315738952718675},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.037599156028591096},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.04121247411239892},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.04656287899706513},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.038520594011060894},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.04027079709339887},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.048373074969276786},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.037970980047248304},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.044722256949171424},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.048211664077825844},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.03928297501988709},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.040895221987739205},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.047834259923547506},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.040038152947090566},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.04173213196918368},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.04809356399346143},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.04052638402208686},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.046114103053696454},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.04008367797359824},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.04100462608039379},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.04753320896998048},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.03898396797012538},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.040894018951803446},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.047692765947431326},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.038160532945767045},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.04187595099210739},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.046314134960994124},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.03853361797519028},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.04191875993274152},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.052512442925944924},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.037831870024092495},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.04208830592688173},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.0476224459707737},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.03937575803138316},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.042271347949281335},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.04919962596613914},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.04217080993112177},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.04887280706316233},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.03945167001802474},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.04278493602760136},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.049006334971636534},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.04149927792605013},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.03981589002069086},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.047538858954794705},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.03820846299640834},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.04201502003706992},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.04733552993275225},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.03850487503223121},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.04162366990931332},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.04760560393333435},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.04054885602090508},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.042009740020148456},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.05057174398098141},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.042170408996753395},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.05230536707676947},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.04147290904074907},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.0394966839812696},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.04749598994385451},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.03961404296569526},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.04380234202835709},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.04983794700819999},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.041420731926336884},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.043693979969248176},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.047647552099078894},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.03922238294035196},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.045582561986520886},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.05021383706480265},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.0421932419994846},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.05023722106125206},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.041097519104368985},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.044117970042862},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.05236917606089264},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.0415588190080598},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.043844109983183444},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.05050939891953021},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.04314521700143814},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.05102583102416247},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.04243899101857096},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.043507943977601826},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.05155154096428305},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.04307421506382525},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.047254848992452025},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.05074263294227421},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.04298452800139785},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.051630988949909806},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.04459033103194088},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.045416492968797684},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.05062233901116997},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.04179630195721984},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.04450318997260183},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.049208643031306565},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.043300407007336617},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.05118735204450786},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.04783927998505533},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.04517059295903891},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.05231138796079904},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.043515947996638715},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.04534765402786434},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.050866393954493105},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.04508395295124501},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.053075813106261194},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.04197234008461237},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.04703016090206802},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.05320713808760047},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.04455108905676752},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.051063240971416235},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.045963943004608154},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.04569078399799764},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.04993445589207113},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.04143815394490957},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.04468885494861752},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.051313778036274016},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.04357909702230245},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.05459296202752739},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.04553305997978896},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.04915870993863791},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.05529620300512761},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.04373314301483333},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.05510335392318666},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.043355738976970315},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.04598731594160199},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.05152773694135249},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.04419892199803144},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.05942033289466053},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.04913498600944877},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.05117376602720469},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.053760229027830064},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.048335173982195556},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.057390418951399624},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.04684559698216617},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.04898199008312076},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.054091908037662506},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.05009058699943125},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.05586344201583415},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.049318338045850396},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.05838256806600839},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.04771152092143893},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.05099035508465022},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.05459632910788059},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.050714580924250185},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.05882647598627955},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.05021212401334196},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.061589327990077436},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.04853502695914358},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.05167367891408503},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.055715095018967986},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.049568974995054305},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.05460717901587486},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.051328416098840535},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.060179288033396006},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.04900245799217373},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.051759889931418},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.055012735072523355},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.05157619796227664},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.05466239107772708},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.04897727200295776},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.05524497805163264},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.04622792499139905},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.051828338066115975},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.06053537200205028},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.05332399602048099},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.06035382393747568},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.04810950404498726},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.060051981010474265},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.05104772199410945},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.05899832595605403},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.046205524005927145},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.05106767895631492},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.05649500899016857},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.052726902067661285},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.06121602200437337},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.05203595501370728},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.06006858195178211},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.054237128933891654},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.06398580595850945},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.053685641032643616},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.05534215900115669},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.059020777931436896},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.05574405810330063},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.058884063037112355},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.053875284967944026},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.0612333039753139},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.058328087092377245},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.06456067692488432},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.05043457797728479},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.06516766897402704},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.05452658806461841},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.06341563200112432},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.051373429014347494},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.06056159094441682},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.054085626965388656},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.05631020397413522},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.06058188888709992},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.05987318803090602},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.06280779896769673},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.056967219919897616},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.060808571986854076},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.05652311095036566},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.06136452907230705},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.05232475302182138},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.06444818794261664},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.05438319104723632},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.06508275109808892},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.05700815503951162},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.06494905089493841},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.05476236797403544},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.06343418802134693},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.057637218036688864},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.06819219898898154},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.06315856298897415},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.06943505606614053},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.05922340601682663},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.06642704689875245},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.06365802499931306},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.06887436099350452},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.059296612977050245},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.06432065006811172},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.06238362903241068},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.06912263401318341},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.06220866192597896},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.06784961000084877},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.06339625595137477},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.06582767993677408},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.06953468197025359},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.06074940098915249},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.06841868197079748},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.06418047903571278},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.06915697699878365},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.05959290498867631},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.0682363510131836},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.06155090592801571},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.06891490600537509},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.06245706498157233},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.07215061993338168},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.06445152510423213},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.06648937391582876},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.06177384196780622},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.06852293503470719},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.0634379539405927},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.06550584000069648},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.0688053930643946},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.06218545895535499},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.06540742598008364},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.06050763092935085},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.06515936297364533},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.060428353026509285},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.07024371391162276},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.05831330898217857},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.07015647098887712},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.05781570100225508},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.06950164993759245},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.05853720707818866},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.06726866704411805},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.058123163995333016},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.06546180695295334},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.05870087200310081},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.06912730203475803},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.05835860292427242},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.06661056005395949},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.06711499998345971},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.07393339404370636},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.07002881297376007},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.07678279711399227},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.07821254304144531},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.06877701997291297},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.0756287039257586},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.06920115102548152},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.07338802702724934},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.08058098796755075},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.07189880998339504},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.08035471709445119},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.06776960100978613},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.07713018404319882},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.0783869280712679},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.07232485397253186},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.08165987103711814},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.0823923860443756},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.07271958992350847},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.08220353408250958},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.07562885491643101},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.07878787501249462},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.08237377204932272},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.07289888500235975},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.08077966899145395},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.08971864695195109},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.07269490498583764},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.08177626703400165},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.08286569791380316},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.0750980949960649},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.07932590798009187},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.07436383608728647},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.08041766402311623},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.08699918596539646},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.07177465897984803},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.08197042997926474},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.0929454939905554},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.08201540308073163},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.08819467399735004},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.08680139703210443},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.09123886097222567},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.08600268710870296},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.08903867891058326},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.09154345898423344},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.08133005502168089},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.09082216396927834},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.09241893095895648},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.08986472908873111},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.08419205201789737},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.08879821992013603},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.0927347899414599},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.08713582996279001},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.09455630602315068},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.09379013697616756},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.09617930301465094},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.08711114502511919},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.09703071101102978},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.09507877100259066},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.09822487703058869},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.0927398590138182},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.09458467899821699},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.09719815209973603},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.11093455401714891},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.11183775903191417},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.11387632007244974},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.11358162097167224},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.11009244399610907},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.12555016693659127},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.1192304230062291},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.12637701909989119},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.13543251296505332},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.15514427702873945},{"candidates":13,"kind":"choice","padded_tokens":2171,"seconds":0.16695503890514374}],"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"sub1b-arc-easy-standard-v2","finished":1792205767,"gpu":0,"id":"13bc7aefda65a9714768d863","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-0.5B-Instruct","model_revision":"7ae557604adf67be50417f59c2c2f167def9a775","n":2376,"notes":"Full ARC-Easy test split (2376 examples); FP32, 1024-token input ceiling, raw summed answer-token log likelihood, zero-shot Question/Answer prompt. Character-normalized accuracy reported separately as acc_norm. Native masked-padding adapter uses unpadded row forwards. Distinct from the earlier token-normalized pilot.","peak_gpu_bytes":2448489472,"profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Easy (full test split, raw accuracy)","catalogue_id":"arc-easy","choice_score":"sum_log_likelihood","contract":"Question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Easy","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"sub1b-arc-easy-full","license":["cc-by-sa-4.0"],"limit":2376,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Easy / full sub-1B standard evaluation","parent_task":"reading","protocol_version":"codesota-sub1b-arc-easy-standard-v2","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Easy/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_sub1b_standard.py","selection":"Full pinned test split: 2376 examples; raw summed conditional answer likelihood, zero-shot Question/Answer prompt; separate character-normalized diagnostic","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":33.362681315047666,"measurement_version":"gpu-slot-v1"},"runner_sha256":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","scope":"cf50d4b44de811db4aed984c74f66a1eab461af452494ae6c905f9c80d3ffc25","score":0.6557239057239057,"scoring":"argmax raw summed conditional answer-token log likelihood; zero-shot Question/Answer prompt; no chat template or added BOS","seconds":30.392575137899257,"seed":42,"selection":"full pinned ARC-Easy test split; all 2376 examples in source order","status":"completed","task":"sub1b-arc-easy-full","worker":"white-gpu0"}